From e20b7de515945b14658f91cd7c26a3361d853c36 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 27 Aug 2026 11:05:03 +0200 Subject: [PATCH 1/3] Configure and apply Java formatting Co-authored-by: Codex GPT-5 --- .gitattributes | 2 + .github/workflows/check_java_formatting.yml | 25 + README.md | 25 + .../java/com/bakdata/conquery/AutoDoc.java | 23 +- .../java/com/bakdata/conquery/Constants.java | 288 ++++--- .../com/bakdata/conquery/handler/Ctx.java | 6 +- .../conquery/handler/GroupHandler.java | 180 +++-- .../conquery/handler/SimpleWriter.java | 26 +- .../AbstractJavadocIntrospection.java | 21 +- .../AbstractNodeWithMemberIntrospection.java | 28 +- .../introspection/EnumIntrospection.java | 11 +- .../conquery/introspection/Introspection.java | 24 +- .../introspection/SimpleIntrospection.java | 2 +- .../java/com/bakdata/conquery/model/Base.java | 5 +- .../com/bakdata/conquery/model/Group.java | 5 +- .../java/com/bakdata/conquery/Conquery.java | 12 +- .../conquery/apiv1/AdditionalMediaTypes.java | 2 +- .../com/bakdata/conquery/apiv1/ApiV1.java | 174 ++-- .../conquery/apiv1/FilterTemplate.java | 2 +- .../conquery/apiv1/FormConfigPatch.java | 81 +- .../com/bakdata/conquery/apiv1/KeyValue.java | 4 +- .../bakdata/conquery/apiv1/MeProcessor.java | 35 +- .../bakdata/conquery/apiv1/MetaDataPatch.java | 138 ++-- .../conquery/apiv1/QueryProcessor.java | 341 ++++---- .../apiv1/RequestAwareUriBuilder.java | 1 + .../bakdata/conquery/apiv1/RequestHelper.java | 20 +- .../apiv1/auth/PasswordHashCredential.java | 3 +- .../conquery/apiv1/auth/ProtoRole.java | 4 +- .../conquery/apiv1/auth/ProtoUser.java | 172 ++-- .../apiv1/auth/UsernamePasswordToken.java | 3 +- .../conquery/apiv1/forms/ExternalForm.java | 19 +- .../bakdata/conquery/apiv1/forms/Form.java | 9 +- .../conquery/apiv1/forms/FormConfigAPI.java | 14 +- .../conquery/apiv1/forms/IndexPlacement.java | 9 +- .../apiv1/forms/export_form/AbsoluteMode.java | 9 +- .../forms/export_form/EntityDateMode.java | 136 ++-- .../apiv1/forms/export_form/ExportForm.java | 59 +- .../forms/export_form/FullExportForm.java | 19 +- .../apiv1/forms/export_form/RelativeMode.java | 7 +- .../apiv1/frontend/FrontendConfiguration.java | 12 +- .../frontend/FrontendFilterConfiguration.java | 7 +- .../conquery/apiv1/frontend/FrontendList.java | 7 +- .../conquery/apiv1/frontend/FrontendNode.java | 4 +- .../apiv1/frontend/FrontendResultType.java | 4 +- .../conquery/apiv1/frontend/FrontendRoot.java | 2 +- .../apiv1/frontend/FrontendTable.java | 5 +- .../apiv1/frontend/FrontendValidityDate.java | 1 - .../apiv1/frontend/FrontendValue.java | 4 +- .../apiv1/frontend/VersionContainer.java | 6 +- .../apiv1/query/ArrayConceptQuery.java | 24 +- .../conquery/apiv1/query/CQElement.java | 4 +- .../conquery/apiv1/query/ConceptQuery.java | 11 +- .../conquery/apiv1/query/ExternalUpload.java | 5 +- .../bakdata/conquery/apiv1/query/Query.java | 11 +- .../apiv1/query/QueryDescription.java | 61 +- .../conquery/apiv1/query/ResultHeaders.java | 6 +- .../apiv1/query/SecondaryIdQuery.java | 28 +- .../apiv1/query/TableExportQuery.java | 137 ++-- .../apiv1/query/TemporalSamplerFactory.java | 8 +- .../apiv1/query/concept/filter/CQTable.java | 14 +- .../query/concept/filter/FilterValue.java | 8 +- .../concept/filter/ValidityDateContainer.java | 2 +- .../apiv1/query/concept/specific/CQAnd.java | 13 +- .../query/concept/specific/CQConcept.java | 105 +-- .../concept/specific/CQDateRestriction.java | 8 +- .../query/concept/specific/CQNegation.java | 4 +- .../apiv1/query/concept/specific/CQOr.java | 13 +- .../query/concept/specific/CQReusedQuery.java | 11 +- .../query/concept/specific/CQTemporal.java | 55 +- .../concept/specific/external/CQExternal.java | 93 ++- .../concept/specific/external/DateFormat.java | 11 +- .../specific/external/EntityResolver.java | 22 +- .../specific/external/EntityResolverUtil.java | 43 +- .../conquery/commands/ConqueryCommand.java | 149 ++-- .../DistributedStandaloneCommand.java | 15 +- .../conquery/commands/ManagerNode.java | 75 +- .../conquery/commands/MigrateCommand.java | 161 ++-- .../commands/PreprocessorCommand.java | 750 +++++++++--------- .../conquery/commands/RecodeStoreCommand.java | 99 ++- .../conquery/commands/ShardCommand.java | 7 +- .../bakdata/conquery/commands/ShardNode.java | 30 +- .../conquery/commands/StandaloneCommand.java | 1 + .../internationalization/CQElementC10n.java | 6 +- .../DateContextResolutionC10n.java | 6 +- .../ExcelSheetNameC10n.java | 6 +- .../internationalization/ExportFormC10n.java | 3 +- .../ResultHeadersC10n.java | 4 +- .../com/bakdata/conquery/io/cps/CPSBase.java | 6 +- .../com/bakdata/conquery/io/cps/CPSMap.java | 20 +- .../com/bakdata/conquery/io/cps/CPSType.java | 6 +- .../conquery/io/cps/CPSTypeIdResolver.java | 70 +- .../external/form/ExternalFormBackendApi.java | 68 +- .../io/external/form/ExternalTaskState.java | 7 +- .../io/external/form/FormBackendVersion.java | 4 +- .../conquery/io/freemarker/Freemarker.java | 21 +- .../io/jackson/ConquerySerializersModule.java | 8 +- .../conquery/io/jackson/Initializing.java | 4 +- .../conquery/io/jackson/Injectable.java | 13 +- .../bakdata/conquery/io/jackson/Jackson.java | 81 +- .../conquery/io/jackson/JacksonUtil.java | 16 +- .../io/jackson/MutableInjectableValues.java | 20 +- .../com/bakdata/conquery/io/jackson/View.java | 18 +- .../DefaultSocketSessionConfigMixIn.java | 10 +- .../io/jackson/mixin/Object2IntMapMixIn.java | 27 +- .../serializer/BitSetDeserializer.java | 5 +- .../serializer/CBlockDeserializer.java | 18 +- .../serializer/CDateRangeDeserializer.java | 22 +- .../serializer/CDateSetDeserializer.java | 42 +- .../serializer/CDateSetSerializer.java | 4 +- .../CharArrayBufferDeserializer.java | 14 +- .../serializer/CharArrayBufferSerializer.java | 6 +- .../ClassToInstanceMapDeserializer.java | 19 +- .../serializer/ConqueryDoubleSerializer.java | 7 +- .../serializer/CurrencyUnitDeserializer.java | 3 +- .../serializer/DatasetParamInjector.java | 15 +- .../serializer/FormatedDateDeserializer.java | 26 +- .../io/jackson/serializer/IdDeserializer.java | 48 +- .../jackson/serializer/IdKeyDeserializer.java | 6 +- .../Int2IntArrayMapDeserializer.java | 12 +- .../serializer/LocaleDeserializer.java | 5 +- .../serializer/MediaTypeDeserializer.java | 11 +- .../conquery/io/jersey/ExtraMimeTypes.java | 4 +- .../jersey/IdPathParamConverterProvider.java | 23 +- .../conquery/io/jersey/LocaleFilter.java | 9 +- .../io/jersey/MessagePackProvider.java | 35 +- .../conquery/io/jersey/RESTServer.java | 5 +- .../io/jetty/CORSPreflightRequestFilter.java | 8 +- .../conquery/io/jetty/CORSResponseFilter.java | 5 +- .../conquery/io/jetty/CachingFilter.java | 13 +- .../jetty/ConqueryErrorExceptionMapper.java | 18 +- .../io/jetty/ConqueryJsonExceptionMapper.java | 15 +- .../jetty/IllegalArgumentExceptionMapper.java | 7 +- .../jetty/JsonValidationExceptionMapper.java | 30 +- .../jetty/NoSuchElementExceptionMapper.java | 8 +- .../bakdata/conquery/io/mina/MdcFilter.java | 6 +- .../conquery/io/mina/MessageSender.java | 36 +- .../conquery/io/mina/NetworkSession.java | 22 +- .../jackson/AsyncJacksonProtocolFilter.java | 29 +- .../io/mina/jackson/AsyncPublisher.java | 9 +- .../conquery/io/mina/jackson/AsyncReader.java | 3 +- .../io/result/ExternalExecutionInfo.java | 4 +- .../ResultRender/ResultRendererProvider.java | 8 +- .../conquery/io/result/ResultUtil.java | 23 +- .../io/result/arrow/ArrowRenderer.java | 124 +-- .../conquery/io/result/arrow/ArrowUtil.java | 34 +- .../io/result/arrow/ResultArrowProcessor.java | 114 +-- .../conquery/io/result/arrow/RowConsumer.java | 13 +- .../conquery/io/result/csv/CsvRenderer.java | 40 +- .../io/result/csv/ResultCsvProcessor.java | 42 +- .../io/result/excel/ExcelRenderer.java | 73 +- .../io/result/excel/ResultExcelProcessor.java | 34 +- .../json/ResultJsonDescriptionProcessor.java | 30 +- .../io/result/parquet/ColumnConsumer.java | 2 +- .../parquet/EntityResultWriteSupport.java | 32 +- .../io/result/parquet/ParquetRenderer.java | 23 +- .../parquet/ResultParquetProcessor.java | 32 +- .../io/storage/IdentifiableStore.java | 62 +- .../conquery/io/storage/ManagedStore.java | 2 + .../conquery/io/storage/MetaStorage.java | 427 +++++----- .../conquery/io/storage/NamespaceStorage.java | 358 ++++----- .../io/storage/NamespacedStorageImpl.java | 7 +- .../bakdata/conquery/io/storage/Store.java | 98 +-- .../conquery/io/storage/StoreMappings.java | 42 +- .../io/storage/WorkerStorageImpl.java | 20 +- .../io/storage/xodus/stores/BigStore.java | 89 ++- .../io/storage/xodus/stores/CachedStore.java | 37 +- .../xodus/stores/EnvironmentRegistry.java | 11 +- .../xodus/stores/KeyIncludingStore.java | 111 ++- .../xodus/stores/SerializingStore.java | 184 +++-- .../io/storage/xodus/stores/StoreInfo.java | 5 +- .../io/storage/xodus/stores/XodusStore.java | 11 +- .../conquery/metrics/ActiveUsersFilter.java | 16 +- .../conquery/metrics/ExecutionMetrics.java | 22 +- .../bakdata/conquery/metrics/JobMetrics.java | 74 +- .../metrics/prometheus/PrometheusBundle.java | 5 +- .../com/bakdata/conquery/mode/Manager.java | 8 + .../conquery/mode/ManagerProvider.java | 14 +- .../conquery/mode/NamespaceHandler.java | 17 +- .../cluster/ClusterConnectionManager.java | 19 +- .../mode/cluster/ClusterConnectionShard.java | 82 +- .../mode/cluster/ClusterEntityResolver.java | 18 +- .../mode/cluster/ClusterImportHandler.java | 47 +- .../mode/cluster/ClusterManagerProvider.java | 40 +- .../mode/cluster/ClusterNamespaceHandler.java | 42 +- .../conquery/mode/cluster/ClusterState.java | 12 +- .../mode/cluster/ClusterStorageListener.java | 22 +- .../mode/cluster/InternalMapperFactory.java | 12 +- .../mode/local/ConnectionManager.java | 6 +- .../mode/local/LocalManagerProvider.java | 35 +- .../mode/local/LocalNamespaceHandler.java | 59 +- .../mode/local/LocalStorageListener.java | 4 +- .../mode/local/ManagedConnection.java | 27 +- .../mode/local/SqlEntityResolver.java | 59 +- .../mode/local/SqlStorageHandler.java | 18 +- .../mode/local/UpdateMatchingStatsSqlJob.java | 23 +- .../models/auth/AuthorizationController.java | 51 +- .../models/auth/AuthorizationHelper.java | 43 +- .../auth/ConqueryAuthenticationInfo.java | 17 +- .../models/auth/ConqueryAuthenticator.java | 86 +- .../auth/ConqueryAuthorizationRealm.java | 280 +++---- .../conquery/models/auth/UserManageable.java | 82 +- .../models/auth/basic/AccessTokenCreator.java | 2 +- .../models/auth/basic/JWTokenHandler.java | 28 +- .../auth/basic/LocalAuthenticationRealm.java | 89 ++- .../models/auth/basic/PasswordHelper.java | 30 +- ...UserAuthenticationManagementProcessor.java | 96 +-- .../conquerytoken/ConqueryTokenRealm.java | 41 +- .../models/auth/develop/DevAuthConfig.java | 15 +- .../models/auth/develop/DevelopmentToken.java | 4 +- .../auth/develop/FirstInitialUserRealm.java | 19 +- .../auth/develop/UserIdTokenExtractor.java | 8 +- .../models/auth/entities/FilteredUser.java | 6 +- .../models/auth/entities/PermissionOwner.java | 321 ++++---- .../conquery/models/auth/entities/Role.java | 72 +- .../models/auth/entities/RoleOwner.java | 46 +- .../models/auth/entities/Subject.java | 2 +- .../conquery/models/auth/entities/User.java | 458 +++++------ .../oidc/IntrospectionDelegatingRealm.java | 76 +- .../auth/oidc/JwtPkceVerifyingRealm.java | 51 +- .../oidc/keycloak/AccessTokenResponse.java | 6 +- .../auth/oidc/keycloak/ClientCredentials.java | 1 + .../ClientCredentialsGrantRequestFilter.java | 20 +- .../models/auth/oidc/keycloak/GroupUtil.java | 12 +- .../auth/oidc/keycloak/KeycloakApi.java | 18 +- .../auth/oidc/keycloak/KeycloakGroup.java | 16 +- .../IdpDelegatingAccessTokenCreator.java | 29 +- .../models/auth/permissions/Ability.java | 12 +- .../models/auth/permissions/AbilitySets.java | 27 +- .../auth/permissions/AdminPermission.java | 1 - .../auth/permissions/ConceptPermission.java | 8 +- .../auth/permissions/ConqueryPermission.java | 18 +- .../auth/permissions/DatasetPermission.java | 10 +- .../auth/permissions/ExecutionPermission.java | 12 +- .../permissions/FormConfigPermission.java | 8 +- .../auth/permissions/FormPermission.java | 4 +- .../permissions/StringPermissionBuilder.java | 59 +- .../auth/util/SkippingCredentialsMatcher.java | 4 +- .../auth/util/SubjectPrincipalCollection.java | 28 +- .../models/auth/web/AuthCookieFilter.java | 19 +- .../conquery/models/auth/web/AuthFilter.java | 9 +- .../web/AuthenticationExceptionMapper.java | 6 +- .../web/AuthorizationExceptionMapper.java | 6 +- .../auth/web/RedirectingAuthFilter.java | 37 +- .../auth/web/csrf/CsrfTokenCheckFilter.java | 13 +- .../auth/web/csrf/CsrfTokenSetFilter.java | 61 +- .../bakdata/conquery/models/common/CDate.java | 4 +- .../conquery/models/common/CDateSet.java | 98 +-- .../conquery/models/common/CQuarter.java | 4 +- .../conquery/models/common/ColumnUtils.java | 4 +- .../conquery/models/common/IRange.java | 14 +- .../conquery/models/common/QuarterUtils.java | 36 +- .../bakdata/conquery/models/common/Range.java | 34 +- .../models/common/daterange/CDateRange.java | 75 +- .../conquery/models/config/APIConfig.java | 7 +- .../conquery/models/config/ArrowConfig.java | 1 + .../models/config/ArrowResultProvider.java | 25 +- .../conquery/models/config/CSVConfig.java | 19 +- .../conquery/models/config/ClusterConfig.java | 18 +- .../conquery/models/config/ColumnConfig.java | 2 +- .../models/config/ConqueryConfig.java | 32 +- .../models/config/ConqueryMetricsConfig.java | 3 +- .../models/config/CsvResultProvider.java | 13 +- .../config/DatabaseConnectionConfig.java | 5 +- .../conquery/models/config/Dialect.java | 3 +- .../conquery/models/config/ExcelConfig.java | 26 +- .../models/config/ExcelResultProvider.java | 33 +- .../models/config/ExternalResultProvider.java | 12 +- .../models/config/FormBackendConfig.java | 68 +- .../models/config/FrontendConfig.java | 11 +- .../models/config/IdColumnConfig.java | 46 +- .../conquery/models/config/IndexConfig.java | 9 +- .../config/JsonDescriptionResultProvider.java | 11 +- .../conquery/models/config/LocaleConfig.java | 37 +- .../conquery/models/config/ManualConfig.java | 5 - .../conquery/models/config/MinaConfig.java | 11 +- .../models/config/ParquetResultProvider.java | 14 +- .../conquery/models/config/ParserConfig.java | 36 +- .../conquery/models/config/PluginConfig.java | 6 +- .../models/config/PreprocessingConfig.java | 7 +- .../conquery/models/config/QueryConfig.java | 4 +- .../models/config/SqlConnectorConfig.java | 8 +- .../models/config/StandaloneConfig.java | 3 +- .../conquery/models/config/StoreFactory.java | 8 +- .../models/config/ThreadPoolDefinition.java | 14 +- .../models/config/XodusStoreFactory.java | 240 ++++-- .../auth/ApiKeyClientFilterProvider.java | 5 +- .../AuthenticationClientFilterProvider.java | 3 +- .../config/auth/AuthenticationConfig.java | 39 +- .../auth/AuthenticationRealmFactory.java | 7 +- .../config/auth/AuthorizationConfig.java | 3 +- .../auth/DefaultAuthorizationConfig.java | 15 +- .../auth/DevelopmentAuthorizationConfig.java | 29 +- .../IntrospectionDelegatingRealmFactory.java | 26 +- .../auth/JwtPkceVerifyingRealmFactory.java | 187 +++-- .../auth/LocalAuthenticationConfig.java | 82 +- ...OIDCAuthorizationCodeFlowRealmFactory.java | 5 +- ...ceOwnerPasswordCredentialRealmFactory.java | 24 +- .../config/search/InternalSearchConfig.java | 48 +- .../models/config/search/SearchConfig.java | 2 +- .../config/search/solr/FilterValueConfig.java | 134 ++-- .../models/config/search/solr/SolrConfig.java | 66 +- .../conquery/models/datasets/Column.java | 2 +- .../conquery/models/datasets/Dataset.java | 4 +- .../models/datasets/PreviewConfig.java | 49 +- .../conquery/models/datasets/Table.java | 17 +- .../models/datasets/concepts/Concept.java | 23 +- .../datasets/concepts/ConceptElement.java | 3 +- .../models/datasets/concepts/Connector.java | 14 +- .../concepts/DaterangeSelectOrFilter.java | 13 +- .../concepts/FrontEndConceptBuilder.java | 252 +++--- .../datasets/concepts/MatchingStats.java | 4 +- .../models/datasets/concepts/Searchable.java | 3 +- .../datasets/concepts/SelectHolder.java | 2 + .../datasets/concepts/StructureNode.java | 11 +- .../datasets/concepts/ValidityDate.java | 10 +- .../concepts/conditions/AndCondition.java | 26 +- .../concepts/conditions/CTCondition.java | 31 +- .../conditions/ColumnEqualCondition.java | 13 +- .../concepts/conditions/EqualCondition.java | 15 +- .../concepts/conditions/GroovyCondition.java | 24 +- .../concepts/conditions/IsEmptyCondition.java | 5 +- .../conditions/IsPresentCondition.java | 6 +- .../concepts/conditions/NotCondition.java | 12 +- .../concepts/conditions/PrefixCondition.java | 13 +- .../conditions/PrefixRangeCondition.java | 7 +- .../datasets/concepts/filters/Filter.java | 32 +- .../concepts/filters/GroupFilter.java | 1 - .../specific/AbstractSelectFilter.java | 1 + .../filters/specific/CountFilter.java | 12 +- .../filters/specific/CountQuartersFilter.java | 8 +- .../filters/specific/DateDistanceFilter.java | 20 +- .../filters/specific/DurationSumFilter.java | 21 +- .../concepts/filters/specific/FlagFilter.java | 4 +- .../filters/specific/NumberFilter.java | 18 +- .../filters/specific/PrefixTextFilter.java | 7 +- .../specific/QuartersInYearFilter.java | 7 +- .../specific/QueryContextResolvable.java | 2 +- .../filters/specific/SelectFilter.java | 35 +- .../filters/specific/SingleSelectFilter.java | 5 +- .../concepts/filters/specific/SumFilter.java | 28 +- .../datasets/concepts/select/Select.java | 11 +- .../select/concept/ConceptColumnSelect.java | 11 +- .../specific/EventDateUnionSelect.java | 14 +- .../specific/EventDurationSumSelect.java | 2 - .../select/concept/specific/ExistsSelect.java | 10 +- .../concept/specific/QuarterSelect.java | 4 +- .../select/connector/DistinctSelect.java | 4 +- .../select/connector/FirstValueSelect.java | 8 +- .../select/connector/LastValueSelect.java | 5 +- .../select/connector/RandomValueSelect.java | 5 +- .../select/connector/SingleColumnSelect.java | 11 +- .../specific/CountQuartersSelect.java | 6 +- .../connector/specific/CountSelect.java | 6 +- .../specific/DateDistanceSelect.java | 4 +- .../connector/specific/DateUnionSelect.java | 2 - .../connector/specific/DurationSumSelect.java | 118 ++- .../select/connector/specific/FlagSelect.java | 14 +- .../specific/MappableSingleColumnSelect.java | 24 +- .../connector/specific/PrefixSelect.java | 2 - .../specific/QuartersInYearSelect.java | 2 - .../select/connector/specific/SumSelect.java | 32 +- .../concepts/tree/ConceptTreeCache.java | 8 +- .../concepts/tree/ConceptTreeChild.java | 6 +- .../concepts/tree/ConceptTreeConnector.java | 2 +- .../datasets/concepts/tree/TreeConcept.java | 36 +- .../conquery/models/error/ConqueryError.java | 2 +- .../models/error/ConqueryErrorInfo.java | 13 +- .../models/error/SimpleErrorInfo.java | 2 +- .../conquery/models/events/Bucket.java | 27 +- .../conquery/models/events/BucketManager.java | 106 ++- .../conquery/models/events/CBlock.java | 75 +- .../conquery/models/events/EmptyStore.java | 3 +- .../conquery/models/events/MajorTypeId.java | 10 +- .../stores/primitive/StringStoreString.java | 2 +- .../events/stores/root/BooleanStore.java | 3 +- .../events/stores/root/DateRangeStore.java | 2 +- .../models/events/stores/root/DateStore.java | 1 + .../events/stores/root/DecimalStore.java | 2 +- .../models/events/stores/root/MoneyStore.java | 1 + .../models/events/stores/root/RealStore.java | 2 +- .../specific/CompoundDateRangeStore.java | 7 +- .../stores/specific/DirectDateRangeStore.java | 8 +- .../events/stores/specific/MoneyIntStore.java | 7 +- .../specific/QuarterDateRangeStore.java | 5 +- .../stores/specific/RebasingIntegerStore.java | 3 +- .../stores/specific/ScaledDecimalStore.java | 2 +- .../ConceptConfigurationException.java | 12 +- .../models/exceptions/ParsingException.java | 7 +- .../models/exceptions/ValidatorHelper.java | 41 +- .../conquery/models/execution/Labelable.java | 5 +- .../models/execution/ManagedExecution.java | 57 +- .../conquery/models/execution/Owned.java | 2 +- .../conquery/models/execution/Shareable.java | 167 ++-- .../conquery/models/execution/Taggable.java | 1 + .../models/forms/configs/FormConfig.java | 427 +++++----- .../forms/export/AbsExportGenerator.java | 1 + .../forms/export/RelExportGenerator.java | 47 +- .../FormConfigProcessor.java | 387 ++++----- .../frontendconfiguration/FormScanner.java | 80 +- .../ResourceFormConfigProvider.java | 25 +- .../forms/managed/AbsoluteFormQuery.java | 234 +++--- .../forms/managed/AbsoluteFormQueryPlan.java | 6 +- .../models/forms/managed/EntityDateQuery.java | 225 +++--- .../forms/managed/EntityDateQueryPlan.java | 163 ++-- .../forms/managed/ExternalExecution.java | 43 +- .../models/forms/managed/FormQueryPlan.java | 36 +- .../models/forms/managed/ManagedForm.java | 198 ++--- .../forms/managed/ManagedInternalForm.java | 76 +- .../forms/managed/RelativeFormQuery.java | 33 +- .../forms/managed/RelativeFormQueryPlan.java | 35 +- .../conquery/models/forms/util/Alignment.java | 28 +- .../models/forms/util/AlignmentReference.java | 4 +- .../models/forms/util/CalendarUnit.java | 4 +- .../models/forms/util/DateContext.java | 133 ++-- .../models/forms/util/ResultModifier.java | 11 +- .../conquery/models/identifiable/IdMap.java | 210 ++--- .../identifiable/IdResolvingException.java | 6 +- .../models/identifiable/IdentifiableImpl.java | 3 +- .../LabeledNamespaceIdentifiable.java | 8 +- .../NamespacedStorageProvider.java | 6 +- .../conquery/models/identifiable/ids/Id.java | 6 +- .../models/identifiable/ids/IdIterator.java | 19 +- .../models/identifiable/ids/IdUtil.java | 25 +- .../models/identifiable/ids/NamespacedId.java | 2 +- .../identifiable/ids/specific/BucketId.java | 5 +- .../identifiable/ids/specific/CBlockId.java | 5 +- .../identifiable/ids/specific/ColumnId.java | 4 +- .../identifiable/ids/specific/ConceptId.java | 3 +- .../ids/specific/ConceptSelectId.java | 10 +- .../ids/specific/ConceptTreeChildId.java | 9 +- .../ids/specific/ConnectorSelectId.java | 14 +- .../identifiable/ids/specific/FilterId.java | 8 +- .../identifiable/ids/specific/GroupId.java | 100 +-- .../ids/specific/ImportColumnId.java | 2 +- .../identifiable/ids/specific/ImportId.java | 2 +- .../ids/specific/PermissionOwnerId.java | 68 +- .../identifiable/ids/specific/RoleId.java | 100 +-- .../identifiable/ids/specific/SelectId.java | 2 +- .../ids/specific/StructureNodeId.java | 6 +- .../identifiable/ids/specific/UserId.java | 100 +-- .../ids/specific/ValidityDateId.java | 7 +- .../identifiable/mapping/EntityIdMap.java | 5 +- .../identifiable/mapping/EntityPrintId.java | 2 +- .../identifiable/mapping/PrintIdMapper.java | 2 +- .../models/index/FrontendValueIndex.java | 14 +- .../models/index/FrontendValueIndexKey.java | 7 +- .../conquery/models/index/IndexService.java | 145 ++-- .../models/index/InternToExternMapper.java | 3 +- .../conquery/models/index/MapIndexKey.java | 3 +- .../models/index/MapInternToExternMapper.java | 32 +- .../conquery/models/index/MapMultiIndex.java | 3 +- .../models/jobs/CalculateCBlocksJob.java | 21 +- .../com/bakdata/conquery/models/jobs/Job.java | 4 +- .../conquery/models/jobs/JobExecutor.java | 19 +- .../conquery/models/jobs/JobManager.java | 11 +- .../models/jobs/JobManagerStatus.java | 2 +- .../conquery/models/jobs/JobStatus.java | 5 +- .../conquery/models/jobs/ReactingJob.java | 11 +- .../conquery/models/jobs/SimpleJob.java | 2 +- .../models/jobs/UpdateFilterSearchJob.java | 42 +- .../conquery/models/messages/SlowMessage.java | 3 +- .../messages/namespaces/NamespaceMessage.java | 7 +- .../messages/namespaces/WorkerMessage.java | 9 +- .../namespaces/specific/AddImport.java | 2 +- .../specific/CollectColumnValuesJob.java | 1 + .../specific/CollectColumnValuesMessage.java | 115 +-- .../namespaces/specific/ExecuteForm.java | 17 +- .../namespaces/specific/ExecuteQuery.java | 15 +- .../namespaces/specific/ImportBucket.java | 17 +- .../specific/RegisterColumnValues.java | 6 +- .../namespaces/specific/RemoveConcept.java | 8 +- .../namespaces/specific/RemoveImportJob.java | 4 +- .../specific/RemoveSecondaryId.java | 8 +- .../namespaces/specific/RemoveTable.java | 8 +- .../specific/ReportConsistency.java | 27 +- .../namespaces/specific/UpdateConcept.java | 14 +- .../specific/UpdateElementMatchingStats.java | 9 +- .../specific/UpdateMatchingStatsMessage.java | 71 +- .../specific/UpdateSecondaryId.java | 8 +- .../namespaces/specific/UpdateTable.java | 8 +- .../specific/UpdateWorkerBucket.java | 5 +- .../network/MessageToManagerNode.java | 4 +- .../messages/network/MessageToShardNode.java | 4 +- .../messages/network/NetworkMessage.java | 2 +- .../network/NetworkMessageContext.java | 13 +- .../messages/network/specific/AddWorker.java | 9 +- .../network/specific/CancelJobMessage.java | 2 +- .../network/specific/ForwardToNamespace.java | 4 +- .../network/specific/RegisterWorker.java | 15 +- .../network/specific/RemoveWorker.java | 4 +- .../specific/UpdateJobManagerStatus.java | 9 +- .../models/preproc/GroovyPredicate.java | 2 +- .../conquery/models/preproc/PPColumn.java | 2 +- .../conquery/models/preproc/Preprocessed.java | 55 +- .../models/preproc/PreprocessedHeader.java | 24 +- .../models/preproc/PreprocessedReader.java | 6 +- .../models/preproc/PreprocessingJob.java | 2 +- .../conquery/models/preproc/Preprocessor.java | 105 ++- .../models/preproc/TableImportDescriptor.java | 12 +- .../models/preproc/TableInputDescriptor.java | 20 +- .../outputs/CompoundDateRangeOutput.java | 46 +- .../models/preproc/outputs/CopyOutput.java | 15 +- .../preproc/outputs/DateRangeOutput.java | 18 +- .../preproc/outputs/EpochDateRangeOutput.java | 20 +- .../models/preproc/outputs/EpochOutput.java | 7 +- .../models/preproc/outputs/LineOutput.java | 3 +- .../models/preproc/outputs/NullOutput.java | 11 +- .../preproc/outputs/OutputDescription.java | 27 +- .../conquery/models/preproc/package-info.java | 8 +- .../models/preproc/parser/ColumnValues.java | 3 +- .../models/preproc/parser/Parser.java | 6 +- .../parser/specific/BooleanParser.java | 2 +- .../specific/CompoundDateRangeParser.java | 3 +- .../preproc/parser/specific/DateParser.java | 3 +- .../parser/specific/DateRangeParser.java | 4 +- .../parser/specific/IntegerParser.java | 3 +- .../preproc/parser/specific/MoneyParser.java | 4 +- .../preproc/parser/specific/RealParser.java | 4 +- .../preproc/parser/specific/StringParser.java | 2 - .../conquery/models/query/C10nCache.java | 4 +- .../models/query/ColumnDescriptor.java | 7 +- .../models/query/DateAggregationMode.java | 32 +- .../query/DistributedExecutionManager.java | 47 +- .../models/query/ExecutionManager.java | 70 +- .../query/ExternalExecutionInfoImpl.java | 8 +- .../conquery/models/query/ManagedQuery.java | 24 +- .../conquery/models/query/PrintSettings.java | 29 +- .../models/query/QueryExecutionContext.java | 141 ++-- .../conquery/models/query/QueryExecutor.java | 40 +- .../conquery/models/query/QueryJob.java | 6 +- .../models/query/QueryPlanContext.java | 82 +- .../models/query/RequiredEntities.java | 8 +- .../models/query/SingleTableResult.java | 8 +- .../conquery/models/query/Visitable.java | 4 +- .../filter/event/DateDistanceFilterNode.java | 5 +- .../filter/event/MultiSelectFilterNode.java | 6 +- .../filter/event/PrefixTextFilterNode.java | 2 +- .../event/SubstringMultiSelectFilterNode.java | 2 +- .../filter/event/number/NumberFilterNode.java | 3 +- .../query/preview/EntityPreviewExecution.java | 137 ++-- .../query/preview/EntityPreviewForm.java | 94 ++- .../query/preview/EntityPreviewStatus.java | 8 +- .../queryplan/ArrayConceptQueryPlan.java | 16 +- .../query/queryplan/ConceptQueryPlan.java | 27 +- .../queryplan/DateAggregationAction.java | 98 +-- .../query/queryplan/DateAggregator.java | 63 +- .../models/query/queryplan/QPChainNode.java | 3 +- .../models/query/queryplan/QPNode.java | 3 +- .../models/query/queryplan/QPParentNode.java | 13 +- .../models/query/queryplan/QueryPlan.java | 2 +- .../query/queryplan/SecondaryIdQueryPlan.java | 24 +- .../query/queryplan/TableExportQueryPlan.java | 18 +- .../queryplan/aggregators/Aggregator.java | 4 +- .../DistinctValuesWrapperAggregator.java | 10 +- .../aggregators/specific/CountAggregator.java | 1 - .../specific/DateDistanceAggregator.java | 3 +- .../specific/EventDateUnionAggregator.java | 4 +- .../specific/EventDurationSumAggregator.java | 12 +- .../specific/ExistsAggregator.java | 5 +- .../specific/QuarterAggregator.java | 2 +- .../specific/QuartersInYearAggregator.java | 5 +- .../specific/SpecialDateUnion.java | 2 +- .../TwoColumnDurationSumAggregator.java | 1 - .../diffsum/DecimalDiffSumAggregator.java | 8 +- .../diffsum/IntegerDiffSumAggregator.java | 5 +- .../diffsum/MoneyDiffSumAggregator.java | 8 +- .../diffsum/RealDiffSumAggregator.java | 8 +- .../aggregators/specific/package-info.java | 2 +- .../value/ConceptElementsAggregator.java | 8 +- .../value/ConceptValuesAggregator.java | 8 +- .../specific/value/FirstValueAggregator.java | 14 +- .../specific/value/LastValueAggregator.java | 19 +- .../query/queryplan/filter/FilterNode.java | 6 +- .../models/query/queryplan/package-info.java | 2 +- .../query/queryplan/specific/ConceptNode.java | 34 +- .../queryplan/specific/ExternalNode.java | 3 +- .../query/queryplan/specific/FiltersNode.java | 15 +- .../models/query/queryplan/specific/Leaf.java | 2 +- .../queryplan/specific/NegatingNode.java | 2 +- .../query/queryplan/specific/OrNode.java | 6 +- .../queryplan/specific/ValidityDateNode.java | 7 +- .../models/query/queryplan/specific/Yes.java | 2 +- .../specific/temporal/TemporalQueryNode.java | 49 +- .../temporal/TemporalRelationMode.java | 3 +- .../specific/temporal/TemporalSelector.java | 9 +- .../query/resultinfo/ColumnResultInfo.java | 2 +- .../models/query/resultinfo/ResultInfo.java | 9 +- .../models/query/resultinfo/UniqueNamer.java | 7 +- .../printers/ArrowResultPrinters.java | 2 +- .../printers/JavaResultPrinters.java | 2 +- .../printers/JsonResultPrinters.java | 5 +- .../query/resultinfo/printers/Printer.java | 2 +- .../common/DateRangeStringPrinter.java | 17 +- .../printers/common/ListStringPrinter.java | 9 +- .../printers/common/LocalizedEnumPrinter.java | 7 +- .../common/OneToOneMappingPrinter.java | 2 +- .../models/query/results/EntityResult.java | 13 +- .../models/query/results/FormShardResult.java | 19 +- .../query/results/MultilineEntityResult.java | 11 +- .../models/query/results/ShardResult.java | 15 +- .../query/results/SinglelineEntityResult.java | 6 +- .../BooleanColumnStatsCollector.java | 23 +- .../statistics/ColumnStatsCollector.java | 31 +- .../statistics/DateColumnStatsCollector.java | 29 +- .../models/query/statistics/Histogram.java | 67 +- .../HistogramColumnDescription.java | 11 +- .../statistics/ListColumnStatsCollector.java | 2 +- .../NumberColumnStatsCollector.java | 86 +- .../query/statistics/ResultStatistics.java | 182 +++-- .../StringColumnStatsCollector.java | 22 +- .../conquery/models/types/ResultType.java | 108 ++- .../models/worker/ClusterHealthCheck.java | 38 +- .../models/worker/DatasetRegistry.java | 22 +- .../models/worker/DistributedNamespace.java | 43 +- .../models/worker/LocalNamespace.java | 36 +- .../conquery/models/worker/Namespace.java | 40 +- .../models/worker/ShardNodeInformation.java | 5 +- .../conquery/models/worker/ShardWorkers.java | 53 +- .../conquery/models/worker/Worker.java | 38 +- .../conquery/models/worker/WorkerHandler.java | 47 +- .../com/bakdata/conquery/package-info.java | 2 +- .../conquery/resources/ResourceConstants.java | 25 +- .../conquery/resources/ResourcesProvider.java | 5 +- .../admin/AdminPermissionFilter.java | 6 +- .../resources/admin/AdminServlet.java | 175 ++-- .../resources/admin/ShutdownTask.java | 2 +- .../admin/rest/AdminConceptsResource.java | 2 +- .../admin/rest/AdminDatasetProcessor.java | 141 ++-- .../admin/rest/AdminDatasetResource.java | 28 +- .../admin/rest/AdminDatasetsResource.java | 2 +- .../resources/admin/rest/AdminProcessor.java | 60 +- .../resources/admin/rest/AdminResource.java | 62 +- .../admin/rest/AdminTablesResource.java | 12 +- .../admin/rest/AuthOverviewResource.java | 20 +- .../admin/rest/ConfigApiResource.java | 4 +- .../resources/admin/rest/GroupResource.java | 4 +- .../admin/rest/PermissionResource.java | 13 +- .../resources/admin/rest/RoleResource.java | 2 +- .../resources/admin/rest/UIProcessor.java | 218 ++--- .../UserAuthenticationManagementResource.java | 4 +- .../resources/admin/rest/UserResource.java | 4 +- .../resources/admin/ui/AdminUIResource.java | 39 +- .../admin/ui/AuthOverviewUIResource.java | 5 +- .../admin/ui/ConceptsUIResource.java | 8 +- .../admin/ui/DatasetsUIResource.java | 88 +- .../resources/admin/ui/GroupUIResource.java | 15 +- .../admin/ui/IndexServiceUIResource.java | 11 +- .../resources/admin/ui/RoleUIResource.java | 11 +- .../resources/admin/ui/TablesUIResource.java | 12 +- .../resources/admin/ui/UserUIResource.java | 13 +- .../admin/ui/model/ConnectorUIResource.java | 8 +- .../admin/ui/model/FrontendAuthOverview.java | 2 +- .../admin/ui/model/FrontendGroupContent.java | 2 +- .../admin/ui/model/FrontendPermission.java | 10 +- .../admin/ui/model/FrontendRoleContent.java | 2 +- .../admin/ui/model/FrontendUserContent.java | 2 +- .../admin/ui/model/ImportStatistics.java | 7 +- .../admin/ui/model/TableStatistics.java | 5 +- .../resources/admin/ui/model/UIContext.java | 11 +- .../resources/admin/ui/model/UIView.java | 8 +- .../resources/api/ConceptResource.java | 11 +- .../resources/api/ConceptsProcessor.java | 94 ++- .../resources/api/ConfigResource.java | 20 +- .../api/DatasetFormConfigResource.java | 13 +- .../resources/api/DatasetFormResource.java | 12 +- .../resources/api/DatasetQueryResource.java | 52 +- .../resources/api/DatasetsResource.java | 2 +- .../resources/api/EntityPreviewRequest.java | 2 +- .../resources/api/FilterResource.java | 28 +- .../resources/api/FormConfigResource.java | 11 +- .../conquery/resources/api/QueryResource.java | 41 +- .../resources/api/ResultArrowResource.java | 86 +- .../resources/api/ResultCsvResource.java | 46 +- .../resources/api/ResultExcelResource.java | 44 +- .../resources/api/ResultExternalResource.java | 42 +- .../api/ResultJsonDescriptionResource.java | 21 +- .../resources/api/ResultParquetResource.java | 44 +- .../resources/hierarchies/HAuthorized.java | 3 +- .../hierarchies/HierarchyHelper.java | 22 +- .../resources/unprotected/AuthServlet.java | 10 +- .../resources/unprotected/LoginResource.java | 9 +- .../resources/unprotected/TokenResource.java | 10 +- .../sql/conquery/SqlExecutionManager.java | 54 +- .../sql/conquery/SqlMatchingStats.java | 122 ++- .../conquery/sql/conversion/Context.java | 4 +- .../conquery/sql/conversion/Conversions.java | 3 +- .../sql/conversion/NodeConversions.java | 43 +- .../sql/conversion/SharedAliases.java | 28 +- .../conversion/cqelement/CQAndConverter.java | 16 +- .../cqelement/CQDateRestrictionConverter.java | 9 +- .../cqelement/CQExternalConverter.java | 120 ++- .../cqelement/CQNegationConverter.java | 21 +- .../conversion/cqelement/CQOrConverter.java | 16 +- .../conversion/cqelement/CQYesConverter.java | 15 +- .../cqelement/ConversionContext.java | 5 +- .../aggregation/AnsiSqlDateAggregator.java | 90 ++- .../aggregation/DateAggregationContext.java | 15 +- .../aggregation/DateAggregationCte.java | 3 +- .../aggregation/DateAggregationCteStep.java | 41 +- .../aggregation/DateAggregationDates.java | 32 +- .../aggregation/IntermediateTableCte.java | 19 +- .../IntersectAggregationAction.java | 28 +- .../cqelement/aggregation/InvertCte.java | 53 +- .../aggregation/MergeAggregateAction.java | 17 +- .../cqelement/aggregation/MergeCte.java | 16 +- .../aggregation/NodeNoOverlapCte.java | 34 +- .../cqelement/aggregation/OverlapCte.java | 24 +- .../cqelement/aggregation/RowNumberCte.java | 18 +- .../aggregation/SqlAggregationAction.java | 8 +- .../concept/AggregationFilterCte.java | 52 +- .../concept/AggregationSelectCte.java | 30 +- .../cqelement/concept/CQConceptConverter.java | 294 ++++--- .../cqelement/concept/CTConditionContext.java | 8 +- .../cqelement/concept/ConceptCteStep.java | 18 +- .../cqelement/concept/ConceptSqlTables.java | 8 +- .../cqelement/concept/ConnectorCte.java | 11 +- .../cqelement/concept/ConnectorSqlTables.java | 14 +- .../cqelement/concept/FilterContext.java | 6 +- .../concept/IntervalPackingSelectsCte.java | 92 ++- .../cqelement/concept/JoinBranchesCte.java | 54 +- .../cqelement/concept/PreprocessingCte.java | 76 +- .../cqelement/concept/TablePath.java | 93 ++- .../intervalpacking/AggregationMode.java | 3 +- .../AnsiSqlIntervalPacker.java | 116 +-- .../IntervalPackingContext.java | 1 - .../IntervalPackingCteStep.java | 16 +- .../sql/conversion/dialect/DialectBundle.java | 52 +- .../sql/conversion/dialect/Interval.java | 7 +- .../conversion/dialect/SqlDateAggregator.java | 14 +- .../dialect/SqlFunctionProvider.java | 72 +- .../clickhouse/ClickhouseDialectBundle.java | 1 - .../ClickhouseDistinctSelectConverter.java | 31 +- .../ClickhouseFunctionProvider.java | 627 +++++++-------- .../ClickhouseResultSetProcessor.java | 9 +- .../ClickhouseStratificationFunctions.java | 28 +- .../dialect/hana/HanaSqlFunctionProvider.java | 128 +-- .../hana/HanaStratificationFunctions.java | 52 +- .../forms/AbsoluteStratification.java | 178 +++-- .../CombinationNotSupportedException.java | 5 +- .../sql/conversion/forms/FormConstants.java | 28 +- .../sql/conversion/forms/FormCteStep.java | 23 +- .../sql/conversion/forms/FormType.java | 4 +- .../conquery/sql/conversion/forms/Offset.java | 5 +- .../forms/RelativeStratification.java | 286 ++++--- .../forms/StratificationFunctions.java | 107 ++- .../forms/StratificationTableFactory.java | 35 +- .../sql/conversion/model/ColumnDateRange.java | 26 +- .../conversion/model/ConqueryJoinType.java | 4 +- .../sql/conversion/model/CteStep.java | 17 +- .../sql/conversion/model/NameGenerator.java | 6 +- .../sql/conversion/model/NumberMapUtil.java | 12 +- .../sql/conversion/model/QualifyingUtil.java | 4 +- .../sql/conversion/model/QueryStep.java | 41 +- .../sql/conversion/model/QueryStepJoiner.java | 249 +++--- .../model/QueryStepTransformer.java | 55 +- .../sql/conversion/model/Selects.java | 87 +- .../sql/conversion/model/SqlIdColumns.java | 50 +- .../model/StratificationSqlIdColumns.java | 85 +- .../CountQuartersSqlAggregator.java | 172 ++-- .../model/aggregator/CountSqlAggregator.java | 72 +- .../aggregator/DateDistanceSqlAggregator.java | 70 +- .../aggregator/DurationSumSqlAggregator.java | 15 +- .../model/aggregator/FlagSqlAggregator.java | 101 +-- .../model/aggregator/SumSqlAggregator.java | 241 +++--- .../filter/AbstractSelectFilterConverter.java | 23 +- .../model/filter/ConditionUtil.java | 6 +- .../model/filter/FlagCondition.java | 9 +- .../model/filter/MultiSelectCondition.java | 13 +- .../model/filter/NumberFilterConverter.java | 5 +- .../model/filter/RangeCondition.java | 2 +- .../model/filter/WhereCondition.java | 4 +- .../select/ConceptColumnSelectConverter.java | 118 +-- .../select/DateUnionSelectConverter.java | 11 +- .../model/select/DaterangeSelectUtil.java | 135 ++-- .../model/select/DistinctSelectConverter.java | 92 ++- .../select/EventDateUnionSelectConverter.java | 31 +- .../EventDurationSumSelectConverter.java | 40 +- .../model/select/ExistsSelectConverter.java | 8 +- .../model/select/ExistsSqlSelect.java | 12 +- .../model/select/ExtractingSqlSelect.java | 6 +- .../conversion/model/select/FieldWrapper.java | 6 +- .../select/FirstValueSelectConverter.java | 14 +- .../select/LastValueSelectConverter.java | 12 +- .../select/RandomValueSelectConverter.java | 19 +- .../model/select/SelectContext.java | 26 +- .../model/select/SelectConverter.java | 6 +- .../conversion/model/select/SqlSelect.java | 12 +- .../model/select/ValueSelectUtil.java | 129 +-- .../query/AbsoluteFormQueryConverter.java | 50 +- .../query/ConceptQueryConverter.java | 83 +- .../query/EntityDateQueryConverter.java | 59 +- .../query/FormConversionHelper.java | 123 +-- .../query/RelativFormQueryConverter.java | 10 +- .../query/SecondaryIdQueryConverter.java | 9 +- .../query/TableExportQueryConverter.java | 166 ++-- .../sql/execution/DefaultCDateSetParser.java | 18 +- .../execution/DefaultResultSetProcessor.java | 25 +- .../sql/execution/HanaSqlCDateSetParser.java | 22 +- .../sql/execution/ResultSetProcessor.java | 3 +- .../sql/execution/SqlEntityResult.java | 4 +- .../execution/SqlExecutionExecutionInfo.java | 7 +- .../sql/execution/SqlExecutionService.java | 40 +- .../conquery/tasks/LoadStorageTask.java | 18 +- .../conquery/tasks/PermissionCleanupTask.java | 73 +- .../conquery/tasks/QueryCleanupTask.java | 284 +++---- .../conquery/tasks/ReportConsistencyTask.java | 25 +- .../com/bakdata/conquery/util/AuthUtil.java | 22 +- .../conquery/util/CollectionsUtil.java | 2 - .../bakdata/conquery/util/ConqueryEscape.java | 24 +- .../com/bakdata/conquery/util/DateReader.java | 33 +- .../bakdata/conquery/util/NumberParsing.java | 16 +- .../bakdata/conquery/util/PrettyPrinter.java | 27 +- .../com/bakdata/conquery/util/QueryUtils.java | 63 +- .../bakdata/conquery/util/ResourceUtil.java | 31 +- .../conquery/util/SMTPAppenderFactory.java | 7 +- .../conquery/util/TablePrimaryColumnUtil.java | 2 - .../conquery/util/VariableDefaultValue.java | 2 +- .../bakdata/conquery/util/VersionInfo.java | 8 +- .../functions/ChainableUnaryOperator.java | 77 +- .../util/io/ChunkingOutputStream.java | 9 +- .../com/bakdata/conquery/util/io/Cloner.java | 21 +- .../conquery/util/io/ConqueryFileUtil.java | 4 +- .../util/io/EndCheckableInputStream.java | 2 +- .../bakdata/conquery/util/io/FileUtil.java | 38 +- .../conquery/util/io/IdColumnUtil.java | 17 +- .../com/bakdata/conquery/util/io/LogUtil.java | 2 +- .../bakdata/conquery/util/io/ProgressBar.java | 46 +- .../conquery/util/io/package-info.java | 2 +- .../bakdata/conquery/util/package-info.java | 2 +- .../ImmutableProgressReporter.java | 16 +- .../progressreporter/ProgressReporter.java | 9 +- .../ProgressReporterImpl.java | 38 +- .../ProgressReporterUtil.java | 48 +- .../util/progressreporter/package-info.java | 2 +- .../conquery/util/search/SearchProcessor.java | 19 +- .../search/internal/InternalFilterSearch.java | 112 +-- .../util/search/internal/TrieSearch.java | 81 +- .../util/search/solr/ChunkSubmitter.java | 58 +- .../util/search/solr/FilterValueIndexer.java | 20 +- .../util/search/solr/FilterValueSearch.java | 501 ++++++------ .../util/search/solr/SolrEmptySeachable.java | 34 +- .../util/search/solr/SolrProcessor.java | 114 +-- .../solr/entities/SolrFrontendValue.java | 12 +- .../validation/CaffeineSpecValidator.java | 11 +- .../conquery/util/validation/ManualURI.java | 8 +- .../util/validation/ManualURIValidator.java | 26 +- .../util/validation/SqlTableValidator.java | 47 +- .../util/validation/ValidCaffeineSpec.java | 7 +- .../validation/ValidConqueryPermission.java | 8 +- .../util/validation/ValidSqlTable.java | 7 +- .../java/org/apache/hadoop/package-info.java | 2 +- .../api/StoredQueriesProcessorTest.java | 729 +++++++++-------- .../api/form/config/FormConfigTest.java | 212 +++-- .../conquery/api/form/config/TestForm.java | 7 +- .../apiv1/frontend/FrontendValueTest.java | 2 +- .../command/DistributedCommandsTest.java | 58 +- .../conquery/commands/ManagerNodeTest.java | 76 +- .../integration/DownloadLinkGeneration.java | 32 +- .../conquery/integration/IntegrationTest.java | 19 +- .../integration/IntegrationTests.java | 189 +++-- .../conquery/integration/ResourceTree.java | 18 +- .../integration/common/AutoConceptUtil.java | 6 +- .../integration/common/CDateSetTest.java | 277 ++++--- .../integration/common/IntegrationUtils.java | 456 ++++++----- .../integration/common/LoadingUtil.java | 387 +++++---- .../integration/common/PermissionToCheck.java | 3 +- .../integration/common/RequiredColumn.java | 6 +- .../integration/common/RequiredData.java | 6 +- .../common/RequiredSecondaryId.java | 12 +- .../integration/common/RequiredTable.java | 19 +- .../integration/common/RequiredUser.java | 11 +- .../integration/common/ResourceFile.java | 9 +- .../json/AbstractQueryEngineTest.java | 61 +- .../integration/json/ConqueryTestSpec.java | 21 +- .../conquery/integration/json/FormTest.java | 60 +- .../integration/json/JsonIntegrationTest.java | 3 +- .../conquery/integration/json/QueryTest.java | 5 +- .../integration/json/SqlTestDataImporter.java | 15 +- .../integration/json/TestDataImporter.java | 13 +- .../json/WorkerTestDataImporter.java | 28 +- .../json/filter/FilterDescription.java | 8 +- .../integration/json/filter/FilterTest.java | 30 +- .../integration/sql/CsvTableImporter.java | 62 +- .../ClickhouseSqlIntegrationTests.java | 60 +- .../sql/dialect/HanaSqlIntegrationTests.java | 70 +- .../integration/sql/dialect/MockClock.java | 4 +- .../sql/dialect/TestFunctionProvider.java | 20 +- .../sql/dialect/TestSqlConnectorConfig.java | 2 - .../sql/testcontainer/hana/HanaContainer.java | 13 +- .../integration/tests/AdminEndpointTest.java | 8 +- .../tests/AdminUIEndpointTest.java | 8 +- .../integration/tests/ApiEndpointTest.java | 8 +- .../tests/ConceptPermissionTest.java | 175 ++-- .../tests/ConceptResolutionTest.java | 65 +- .../integration/tests/EndpointTestHelper.java | 39 +- .../integration/tests/EntityExportTest.java | 241 +++--- .../integration/tests/EntityResolveTest.java | 48 +- .../tests/ExternalFormBackendTest.java | 172 ++-- .../tests/FilterAutocompleteTest.java | 189 +++-- .../tests/FilterResolutionTest.java | 126 +-- .../integration/tests/GroupHandlingTest.java | 104 ++- .../integration/tests/ImportUpdateTest.java | 86 +- .../tests/MetadataCollectionTest.java | 20 +- .../tests/PermissionGroupHandlingTest.java | 139 ++-- .../tests/PermissionResourceTest.java | 70 +- .../tests/PermissionRoleHandlingTest.java | 168 ++-- .../tests/QueryStatisticsTest.java | 419 ++++++---- .../integration/tests/RestartTest.java | 68 +- .../integration/tests/ReusedQueryTest.java | 119 ++- .../tests/RoleHandlingOnGroupTest.java | 131 +-- .../integration/tests/RoleHandlingTest.java | 125 ++- .../integration/tests/RoleUITest.java | 143 ++-- .../integration/tests/ScriptEndTest.java | 18 +- .../tests/SecondaryIdEndpointTest.java | 181 +++-- .../tests/SqlSchemaValidationTest.java | 187 ++--- .../tests/SuperPermissionTest.java | 115 ++- .../conquery/integration/tests/TestUser.java | 24 +- .../integration/tests/UserErrorTest.java | 57 +- .../ConceptUpdateAndDeletionTest.java | 151 ++-- .../tests/deletion/DatasetDeletionTest.java | 81 +- .../tests/deletion/ImportDeletionTest.java | 153 ++-- .../tests/deletion/TableDeletionTest.java | 119 +-- .../io/AbstractSerializationTest.java | 9 +- .../bakdata/conquery/io/cps/CPSBaseTest.java | 25 +- .../conquery/io/jackson/JacksonTest.java | 43 +- .../ClassToInstanceMapDeserializerTest.java | 4 +- .../serializer/SerializationTestUtil.java | 49 +- .../conquery/io/mina/MinaStackTest.java | 278 +++---- .../conquery/io/result/ResultTestUtil.java | 107 ++- .../arrow/ArrowResultGenerationTest.java | 195 +++-- .../result/csv/CsvResultGenerationTest.java | 85 +- .../result/excel/ExcelResultRenderTest.java | 76 +- .../parquet/ParquetResultGenerationTest.java | 170 ++-- .../storage/xodus/stores/CachedStoreTest.java | 69 +- .../stores/SerializingStoreDumpTest.java | 145 ++-- .../conquery/models/SerializationTests.java | 540 ++++++++----- .../conquery/models/auth/CopyUserTest.java | 7 +- .../IdpDelegatingAccessTokenCreatorTest.java | 65 +- .../InstancePermissionImplificationTest.java | 60 +- .../IntrospectionDelegatingRealmTest.java | 270 ++++--- .../models/auth/LocalAuthRealmTest.java | 55 +- .../conquery/models/auth/OIDCMockServer.java | 94 +-- .../models/auth/PermissionCreationTest.java | 14 +- .../models/auth/oidc/GroupUtilTest.java | 122 ++- .../auth/oidc/IdpConfigRetrievalTest.java | 28 +- .../auth/oidc/JwtPkceVerifyingRealmTest.java | 294 ++++--- .../models/auth/oidc/KeycloakApiTest.java | 8 +- .../conquery/models/common/CQuarterTest.java | 17 +- .../models/common/QuarterUtilsTest.java | 50 +- .../conquery/models/common/RangeTest.java | 138 ++-- .../common/daterange/CDateRangeTest.java | 2 +- .../concepts/filters/TestGroupFilter.java | 43 +- .../InternalFilterSearchItemTest.java | 9 +- .../temporal/TemporalSamplerFactoryTest.java | 48 +- .../concepts/tree/GroovyIndexedTest.java | 72 +- .../models/error/ConqueryErrorTest.java | 12 +- .../stores/primitive/BooleanStoreTest.java | 15 +- .../stores/primitive/IntArrayStoreTest.java | 17 +- .../types/ColumnStoreSerializationTests.java | 62 +- .../events/stores/types/MajorTypesTest.java | 1 - .../models/execution/DefaultLabelTest.java | 74 +- .../models/forms/DateContextTest.java | 529 +++++++++--- .../models/identifiable/IdMapTest.java | 8 +- .../models/identifiable/ids/IdTests.java | 94 ++- .../mapping/PseudomizationTest.java | 4 +- .../models/preproc/PreprocessorTest.java | 2 +- .../parser/specific/DateRangeParserTest.java | 19 +- .../parser/specific/DecimalParserTest.java | 8 +- .../parser/specific/IntegerParserTest.java | 46 +- .../parser/specific/RealParserTest.java | 3 +- .../models/query/DefaultColumnNameTest.java | 239 +++--- .../query/statistics/HistogramTest.java | 12 +- .../conquery/models/types/ResultTypeTest.java | 138 ++-- .../resources/api/ConceptsProcessorTest.java | 5 +- .../resources/api/FilterResourceTest.java | 44 +- .../conquery/service/IndexServiceTest.java | 77 +- .../service/InternalFilterSearchTest.java | 30 +- .../tasks/PermissionCleanupTaskTest.java | 167 ++-- .../conquery/tasks/QueryCleanupTaskTest.java | 389 ++++----- .../conquery/util/ConqueryEscapeTest.java | 44 +- .../conquery/util/NonPersistentStore.java | 222 +++--- .../util/NonPersistentStoreFactory.java | 14 +- .../conquery/util/PasswordHelperTest.java | 16 +- .../util/extensions/GroupExtension.java | 1 + .../util/extensions/MockServerExtension.java | 2 +- .../extensions/NamespaceStorageExtension.java | 2 +- .../util/extensions/SolrServerExtension.java | 9 +- .../extensions/XodusEnvironmentExtension.java | 2 +- .../ProgressReporterTest.java | 32 +- .../ProgressReporterUtilTest.java | 10 +- .../conquery/util/search/QuickSearchTest.java | 96 +-- .../util/search/SolrFilterValueTest.java | 652 ++++++++------- .../conquery/util/support/ConfigOverride.java | 4 +- .../support/ConqueryAuthenticationFilter.java | 2 +- .../util/support/StandaloneSupport.java | 50 +- .../support/TestBootstrappingConquery.java | 20 +- .../conquery/util/support/TestConquery.java | 105 +-- .../util/support/TestLoggingFactory.java | 14 +- .../validation/CaffeineValidationTest.java | 15 +- conquery_eclipse_format_setting.xml | 381 +++++++++ pom.xml | 33 +- 1001 files changed, 24827 insertions(+), 18537 deletions(-) create mode 100644 .gitattributes create mode 100644 .github/workflows/check_java_formatting.yml create mode 100644 conquery_eclipse_format_setting.xml diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000000..6ba5864235b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Keep Spotless formatting stable across operating systems. +*.java text eol=lf diff --git a/.github/workflows/check_java_formatting.yml b/.github/workflows/check_java_formatting.yml new file mode 100644 index 00000000000..5bfc1d1615b --- /dev/null +++ b/.github/workflows/check_java_formatting.yml @@ -0,0 +1,25 @@ +name: "Check Java Formatting" + +on: + push: + branches: + - master + - develop + - release + pull_request: + +jobs: + formatting: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + submodules: true + - name: Set up JDK + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: 21 + cache: maven + - name: Check Java formatting + run: mvn -B -ntp spotless:check diff --git a/README.md b/README.md index c43c0101b03..11da433d4f4 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,31 @@ configurations, such as: ## Development +### Java formatting + +The checked-in `conquery_eclipse_format_setting.xml` is the canonical Java formatter configuration. Spotless also +organizes imports with static imports first. Wildcard imports are currently allowed. Run Spotless from the repository +root: + +```bash +mvn spotless:apply +mvn spotless:check +``` + +Install the repository's pre-push formatting check once per clone: + +```bash +mvn spotless:install-git-pre-push-hook +``` + +The hook formats violations and aborts the push so the resulting changes can be reviewed and committed. It is a local +convenience and can be bypassed; CI remains the authoritative formatting check. + +IntelliJ IDEA does not automatically import an Eclipse formatter profile from the project root. Import +`conquery_eclipse_format_setting.xml` under **Editor | Code Style | Java**, or configure the Adapter for Eclipse Code +Formatter plugin to use the file and select the `conquery` profile. In either case, Maven/Spotless defines the expected +output. + ### Testing #### Integration Tests diff --git a/autodoc/src/main/java/com/bakdata/conquery/AutoDoc.java b/autodoc/src/main/java/com/bakdata/conquery/AutoDoc.java index 252e42b8cb3..52bb49b7a62 100644 --- a/autodoc/src/main/java/com/bakdata/conquery/AutoDoc.java +++ b/autodoc/src/main/java/com/bakdata/conquery/AutoDoc.java @@ -34,17 +34,16 @@ private static void configureJavaParser() { private final ScanResult scan; public AutoDoc() { - scan = new ClassGraph() - .enableAllInfo() - //reject some packages that contain large libraries - .rejectPackages( - "groovy", - "org.codehaus.groovy", - "org.apache", - "org.eclipse", - "com.google" - ) - .scan(); + scan = new ClassGraph().enableAllInfo() + //reject some packages that contain large libraries + .rejectPackages( + "groovy", + "org.codehaus.groovy", + "org.apache", + "org.eclipse", + "com.google" + ) + .scan(); } public void start(File docs) throws IOException { @@ -52,7 +51,7 @@ public void start(File docs) throws IOException { for (Group group : GROUPS) { File target = new File(docs, group.getName().replace(' ', '-') + ".md"); try (var out = new SimpleWriter( - new OutputStreamWriter(new FileOutputStream(target), StandardCharsets.UTF_8) + new OutputStreamWriter(new FileOutputStream(target), StandardCharsets.UTF_8) )) { new GroupHandler(scan, group, out, docs.getCanonicalFile().getParentFile()).handle(); } diff --git a/autodoc/src/main/java/com/bakdata/conquery/Constants.java b/autodoc/src/main/java/com/bakdata/conquery/Constants.java index 3fc2c127fb5..eff9cd32536 100644 --- a/autodoc/src/main/java/com/bakdata/conquery/Constants.java +++ b/autodoc/src/main/java/com/bakdata/conquery/Constants.java @@ -1,12 +1,5 @@ package com.bakdata.conquery; -import java.net.InetAddress; -import java.nio.charset.Charset; -import java.time.ZonedDateTime; -import java.util.Currency; -import java.util.List; -import java.util.Locale; -import java.util.Set; import jakarta.ws.rs.DELETE; import jakarta.ws.rs.GET; import jakarta.ws.rs.POST; @@ -15,6 +8,13 @@ import jakarta.ws.rs.PathParam; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.Response; +import java.net.InetAddress; +import java.nio.charset.Charset; +import java.time.ZonedDateTime; +import java.util.Currency; +import java.util.List; +import java.util.Locale; +import java.util.Set; import com.bakdata.conquery.apiv1.FilterTemplate; import com.bakdata.conquery.apiv1.IdLabel; @@ -86,94 +86,196 @@ import io.dropwizard.util.Duration; public class Constants { - public static final Group[] GROUPS = { - Group.builder().name("Concept JSONs") - .description("Each `*.concept.json` has to contain exactly one [Concept](#Base-Concept).") - .base(new Base(Concept.class, "A concept is a collection of filters and selects and their connection to tables.")) - .base(new Base(CTCondition.class, "These represent guard conditions. A value matches a [ConceptElement](#ConceptElement) if it matches its condition and its parent")) - .base(new Base(Filter.class, "These are used to define filters, than can be used to reduce the result set.")) - .base(new Base(Select.class, "These are used to define selects, that can be used to create additional CSV columns.")) - .otherClass(Connector.class) - .otherClass(KeyValue.class) - .otherClass(ConceptTreeChild.class) - .otherClass(FilterTemplate.class) - .otherClass(ValidityDate.class) - .markerInterface(UniversalSelect.class) - .build(), - Group.builder().name("Import JSONs") - .description("Each `*.import.json` has to contain exactly one [ImportDescriptor](#Type-ImportDescriptor).") - .base(new Base(OutputDescription.class, "")) - .otherClass(TableImportDescriptor.class) - .otherClass(TableInputDescriptor.class) - .build(), - Group.builder().name("Table JSONs") - .description("Each `*.table.json` has to contain exactly one [Tabel](#Type-Tabel).") - .otherClass(Table.class) - .otherClass(Column.class) - .build(), - Group.builder().name("Config JSON") - .description("The `config.json` is required for every type of execution. Its root element is a [ConqueryConfig](#Type-ConqueryConfig) object.") - .base(new Base(AuthorizationConfig.class, "An `AuthorizationConfig` defines the initial users that are created on application startup and other permission related options.")) - .base(new Base(AuthenticationRealmFactory.class, "An `AuthenticationConfig` is used to define how specific realms for authentication are configured.")) - .base(new Base(PluginConfig.class, "A `PluginConfig` is used to define settings for Conquery plugins.")) - .otherClass(APIConfig.class) - .otherClass(ConqueryConfig.class) - .otherClass(ClusterConfig.class) - .otherClass(CSVConfig.class) - .otherClass(FrontendConfig.class) - .otherClass(LocaleConfig.class) - .otherClass(PreprocessingConfig.class) - .otherClass(QueryConfig.class) - .otherClass(StandaloneConfig.class) - .otherClass(XodusStoreFactory.class) - .otherClass(MinaConfig.class) - .otherClass(FrontendConfig.CurrencyConfig.class) - .otherClass(XodusConfig.class) - .otherClasses(List.of(SqlConnectorConfig.class, DatabaseConnectionConfig.class, Dialect.class)) - .hide(Charset.class) - .hide(Currency.class) - .hide(InetAddress.class) - .hide(Locale.class) - .hide(Duration.class) - .hide(DataSize.class) - .build(), - Group.builder().name("REST API JSONs") - .resource(ConfigResource.class) - .resource(DatasetsResource.class) - .resource(DatasetResource.class) - .resource(ConceptResource.class) - .resource(FilterResource.class) - .resource(QueryResource.class) - .resource(ResultCsvResource.class) - .base(new Base(QueryDescription.class, "")) - .base(new Base(CQElement.class, "")) - .base(new Base(FilterValue.class, "")) - .base(new Base(SemanticType.class, "")) + public static final Group[] GROUPS = {Group.builder() + .name("Concept JSONs") + .description( + "Each `*.concept.json` has to contain exactly one [Concept](#Base-Concept).") + .base( + new Base( + Concept.class, + "A concept is a collection of filters and selects and their connection to tables.")) + .base( + new Base( + CTCondition.class, + "These represent guard conditions. A value matches a [ConceptElement](#ConceptElement) if it matches its condition and its parent")) + .base( + new Base( + Filter.class, + "These are used to define filters, than can be used to reduce the result set.")) + .base( + new Base( + Select.class, + "These are used to define selects, that can be used to create additional CSV columns.")) + .otherClass( + Connector.class) + .otherClass(KeyValue.class) + .otherClass( + ConceptTreeChild.class) + .otherClass(FilterTemplate.class) + .otherClass( + ValidityDate.class) + .markerInterface( + UniversalSelect.class) + .build(), Group.builder() + .name( + "Import JSONs") + .description( + "Each `*.import.json` has to contain exactly one [ImportDescriptor](#Type-ImportDescriptor).") + .base( + new Base(OutputDescription.class, "")) + .otherClass( + TableImportDescriptor.class) + .otherClass( + TableInputDescriptor.class) + .build(), Group.builder() + .name( + "Table JSONs") + .description( + "Each `*.table.json` has to contain exactly one [Tabel](#Type-Tabel).") + .otherClass( + Table.class) + .otherClass( + Column.class) + .build(), Group.builder() + .name( + "Config JSON") + .description( + "The `config.json` is required for every type of execution. Its root element is a [ConqueryConfig](#Type-ConqueryConfig) object.") + .base( + new Base( + AuthorizationConfig.class, + "An `AuthorizationConfig` defines the initial users that are created on application startup and other permission related options.")) + .base( + new Base( + AuthenticationRealmFactory.class, + "An `AuthenticationConfig` is used to define how specific realms for authentication are configured.")) + .base( + new Base( + PluginConfig.class, + "A `PluginConfig` is used to define settings for Conquery plugins.")) + .otherClass( + APIConfig.class) + .otherClass( + ConqueryConfig.class) + .otherClass( + ClusterConfig.class) + .otherClass( + CSVConfig.class) + .otherClass( + FrontendConfig.class) + .otherClass( + LocaleConfig.class) + .otherClass( + PreprocessingConfig.class) + .otherClass( + QueryConfig.class) + .otherClass( + StandaloneConfig.class) + .otherClass( + XodusStoreFactory.class) + .otherClass( + MinaConfig.class) + .otherClass( + FrontendConfig.CurrencyConfig.class) + .otherClass( + XodusConfig.class) + .otherClasses( + List.of( + SqlConnectorConfig.class, + DatabaseConnectionConfig.class, + Dialect.class)) + .hide( + Charset.class) + .hide( + Currency.class) + .hide( + InetAddress.class) + .hide( + Locale.class) + .hide( + Duration.class) + .hide( + DataSize.class) + .build(), Group.builder() + .name( + "REST API JSONs") + .resource( + ConfigResource.class) + .resource( + DatasetsResource.class) + .resource( + DatasetResource.class) + .resource( + ConceptResource.class) + .resource( + FilterResource.class) + .resource( + QueryResource.class) + .resource( + ResultCsvResource.class) + .base( + new Base( + QueryDescription.class, + "")) + .base( + new Base( + CQElement.class, + "")) + .base( + new Base( + FilterValue.class, + "")) + .base( + new Base( + SemanticType.class, + "")) - .hide(Response.class) - .hide(ZonedDateTime.class) - .hide(Range.class) + .hide( + Response.class) + .hide( + ZonedDateTime.class) + .hide( + Range.class) - .otherClass(FullExecutionStatus.class) - .otherClass(OverviewExecutionStatus.class) - .otherClass(IdLabel.class) - .otherClass(FrontendConfig.class) - .otherClass(FrontendRoot.class) - .otherClass(FrontendValue.class) - .otherClass(FilterResource.FilterValues.class) - .otherClass(MetaDataPatch.class) - .otherClass(FrontendConfig.CurrencyConfig.class) - .otherClass(ConceptsProcessor.ResolvedFilterResult.class) - .otherClass(FilterResource.AutocompleteRequest.class) - .otherClass(ExecutionStatus.class) - .otherClass(ConceptsProcessor.ResolvedConceptsResult.class) - .otherClass(ConceptResource.ConceptCodeList.class) - .otherClass(CQTable.class) - .otherClass(ValidityDateContainer.class) - .otherClass(FormConfig.class) - .otherClass(FormConfigOverviewRepresentation.class) - .otherClass(FormConfigFullRepresentation.class) - .build() + .otherClass( + FullExecutionStatus.class) + .otherClass( + OverviewExecutionStatus.class) + .otherClass( + IdLabel.class) + .otherClass( + FrontendConfig.class) + .otherClass( + FrontendRoot.class) + .otherClass( + FrontendValue.class) + .otherClass( + FilterResource.FilterValues.class) + .otherClass( + MetaDataPatch.class) + .otherClass( + FrontendConfig.CurrencyConfig.class) + .otherClass( + ConceptsProcessor.ResolvedFilterResult.class) + .otherClass( + FilterResource.AutocompleteRequest.class) + .otherClass( + ExecutionStatus.class) + .otherClass( + ConceptsProcessor.ResolvedConceptsResult.class) + .otherClass( + ConceptResource.ConceptCodeList.class) + .otherClass( + CQTable.class) + .otherClass( + ValidityDateContainer.class) + .otherClass( + FormConfig.class) + .otherClass( + FormConfigOverviewRepresentation.class) + .otherClass( + FormConfigFullRepresentation.class) + .build() }; public static final String JSON_CREATOR = JsonCreator.class.getName(); diff --git a/autodoc/src/main/java/com/bakdata/conquery/handler/Ctx.java b/autodoc/src/main/java/com/bakdata/conquery/handler/Ctx.java index 7d855b3d2c3..536171a0f0b 100644 --- a/autodoc/src/main/java/com/bakdata/conquery/handler/Ctx.java +++ b/autodoc/src/main/java/com/bakdata/conquery/handler/Ctx.java @@ -4,10 +4,12 @@ import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; -import lombok.experimental.Wither; +import lombok.With; @Getter -@Wither @AllArgsConstructor @NoArgsConstructor +@With +@AllArgsConstructor +@NoArgsConstructor public class Ctx { private FieldInfo field; private boolean idOf; diff --git a/autodoc/src/main/java/com/bakdata/conquery/handler/GroupHandler.java b/autodoc/src/main/java/com/bakdata/conquery/handler/GroupHandler.java index 7a36c1973b8..9bd44c7a7d8 100644 --- a/autodoc/src/main/java/com/bakdata/conquery/handler/GroupHandler.java +++ b/autodoc/src/main/java/com/bakdata/conquery/handler/GroupHandler.java @@ -2,6 +2,7 @@ import static com.bakdata.conquery.Constants.*; +import jakarta.ws.rs.core.UriBuilder; import java.io.Closeable; import java.io.File; import java.io.IOException; @@ -15,7 +16,6 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.introspection.Introspection; import com.bakdata.conquery.io.cps.CPSType; @@ -64,7 +64,8 @@ public class GroupHandler { public void handle() throws IOException { out.heading(group.getName()); - out.paragraph("This is an automatically created documentation. It is not 100% accurate since the generator does not handle every edge case."); + out.paragraph( + "This is an automatically created documentation. It is not 100% accurate since the generator does not handle every edge case."); out.paragraph("Instead of a list ConQuery also always accepts a single element."); if (group.getDescription() != null) { out.paragraph(group.getDescription()); @@ -72,23 +73,21 @@ public void handle() throws IOException { for (Base base : group.getBases()) { content.putAll( - base, - scan - .getAllClasses() - .stream() - //all classes that are CPSType - .filter(c -> c.hasAnnotation(CPS_TYPE)) - //resolve multiple CPSType annotations - .flatMap(c -> Arrays - .stream(c.loadClass().getAnnotationsByType(CPSType.class)) - .map(anno -> Pair.of(anno, c)) - ) - //only classes that have the current base as a base - .filter(p -> p.getLeft() - .base() - .equals(base.getBaseClass()) - ) - .collect(Collectors.toList()) + base, + scan.getAllClasses() + .stream() + //all classes that are CPSType + .filter(c -> c.hasAnnotation(CPS_TYPE)) + //resolve multiple CPSType annotations + .flatMap( + c -> Arrays.stream(c.loadClass().getAnnotationsByType(CPSType.class)) + .map( + anno -> Pair.of(anno, c)) + ) + //only classes that have the current base as a base + .filter(p -> p.getLeft().base().equals(base.getBaseClass()) + ) + .collect(Collectors.toList()) ); } @@ -97,7 +96,10 @@ public void handle() throws IOException { } if (!endpoints.isEmpty()) { out.heading("REST endpoints"); - for (Pair endpoint : endpoints.stream().sorted(Comparator.comparing(Pair::getLeft)).toList()) { + for (Pair endpoint : endpoints.stream() + .sorted( + Comparator.comparing(Pair::getLeft)) + .toList()) { handleEndpoint(endpoint.getLeft(), endpoint.getRight()); } } @@ -107,13 +109,21 @@ public void handle() throws IOException { } out.subHeading("Other Types"); - for (Class t : group.getOtherClasses().stream().sorted(Comparator.comparing(Class::getSimpleName)).toList()) { + for (Class t : group.getOtherClasses() + .stream() + .sorted( + Comparator.comparing(Class::getSimpleName)) + .toList()) { handleClass(typeTitle(t), scan.getClassInfo(t.getName())); } if (!group.getMarkerInterfaces().isEmpty()) { out.subHeading("Marker Interfaces"); - for (Class t : group.getMarkerInterfaces().stream().sorted(Comparator.comparing(Class::getSimpleName)).toList()) { + for (Class t : group.getMarkerInterfaces() + .stream() + .sorted( + Comparator.comparing(Class::getSimpleName)) + .toList()) { handleMarkerInterface(markerTitle(t), scan.getClassInfo(t.getName())); } } @@ -130,7 +140,8 @@ private void handleEndpoint(String url, MethodInfo method) throws IOException { } out.line("Expects: " + printType(new Ctx(), param.getTypeSignatureOrTypeDescriptor())); } - out.paragraph("Returns: " + printType(new Ctx(), method.getTypeSignatureOrTypeDescriptor().getResultType())); + out.paragraph( + "Returns: " + printType(new Ctx(), method.getTypeSignatureOrTypeDescriptor().getResultType())); } } @@ -165,13 +176,15 @@ public void handleBase(Base base) throws IOException { out.paragraph(base.getDescription()); final String typeProperty = base.getBaseClass().getAnnotation(JsonTypeInfo.class).property(); - out.paragraph("Different types of " - + base.getBaseClass().getSimpleName() - + " can be used by setting " - + code(typeProperty) - + " to one of the following values:"); + out.paragraph( + "Different types of " + base.getBaseClass().getSimpleName() + " can be used by setting " + code( + typeProperty) + " to one of the following values:"); - for (Pair pair : content.get(base).stream().sorted(Comparator.comparing(p -> p.getLeft().id())).toList()) { + for (Pair pair : content.get(base) + .stream() + .sorted( + Comparator.comparing(p -> p.getLeft().id())) + .toList()) { handleClass(pair.getLeft(), pair.getRight()); } @@ -200,8 +213,7 @@ private void handleClass(String name, ClassInfo c) throws IOException { for (FieldInfo field : c.getFieldInfo().stream().sorted().toList()) { handleField(c, field); } - } - else { + } else { out.paragraph("No fields can be set for this type."); } } @@ -209,9 +221,9 @@ private void handleClass(String name, ClassInfo c) throws IOException { private Closeable details(String name, ClassInfo c, Introspection source) throws IOException { out.subSubHeading( - name - //non-ASCII characters and tags do not change the anchor - + "\u2001" + editLink(source) + "" + name + //non-ASCII characters and tags do not change the anchor + + "\u2001" + editLink(source) + "" ); out.paragraph(source.getDescription()); out.paragraph("
Details

"); @@ -219,9 +231,7 @@ private Closeable details(String name, ClassInfo c, Introspection source) throws out.paragraph("Java Type: " + code(c.getName())); if (!Strings.isNullOrEmpty(source.getExample())) { out.paragraph( - "Example:\n\n```jsonc\n" - + PrettyPrinter.print(source.getExample()) - + "\n```" + "Example:\n\n```jsonc\n" + PrettyPrinter.print(source.getExample()) + "\n```" ); } @@ -237,11 +247,11 @@ private void handleMarkerInterface(String name, ClassInfo c) throws IOException values.add("[" + cl.getSimpleName() + "](" + anchor(typeTitle(cl)) + ")"); } } - content - .values() - .stream() - .filter(p -> c.loadClass().isAssignableFrom(p.getRight().loadClass())) - .forEach(p -> values.add("[" + p.getLeft().id() + "](" + anchor(p.getLeft().id()) + ")")); + content.values() + .stream() + .filter(p -> c.loadClass().isAssignableFrom(p.getRight().loadClass())) + .forEach( + p -> values.add("[" + p.getLeft().id() + "](" + anchor(p.getLeft().id()) + ")")); for (Class cl : group.getMarkerInterfaces()) { if (c.loadClass().isAssignableFrom(cl) && !c.loadClass().equals(cl)) { values.add("[" + cl.getSimpleName() + "](" + anchor(typeTitle(cl)) + ")"); @@ -250,8 +260,7 @@ private void handleMarkerInterface(String name, ClassInfo c) throws IOException if (!values.isEmpty()) { out.paragraph( - "A " + name + " is any of:\n* " - + values.stream().sorted().collect(Collectors.joining("\n* ")) + "A " + name + " is any of:\n* " + values.stream().sorted().collect(Collectors.joining("\n* ")) ); } } @@ -270,12 +279,12 @@ private void handleField(ClassInfo currentType, FieldInfo field) throws IOExcept final String type = printType(ctx, typeSignature); out.table( - editLink(introspec), - name, - type, - findDefault(currentType, field), - introspec.getExample(), - introspec.getDescription() + editLink(introspec), + name, + type, + findDefault(currentType, field), + introspec.getExample(), + introspec.getDescription() ); } @@ -300,8 +309,7 @@ private String findDefault(ClassInfo currentType, FieldInfo field) { final String localPath = Jackson.MAPPER.writeValueAsString(new File(".")); json = StringUtils.replace(json, localPath.substring(1, localPath.length() - 2), "./"); return code(json); - } - catch (Exception e) { + } catch (Exception e) { return "?"; } } @@ -309,11 +317,8 @@ private String findDefault(ClassInfo currentType, FieldInfo field) { private String editLink(Introspection intro) throws IOException { final Path target = root.toPath().relativize(intro.getFile().getCanonicalFile().toPath()); final String line = intro.getLine(); - return "[✎](" - + "https://github.com/bakdata/conquery/edit/develop/" - + FilenameUtils.separatorsToUnix(target.toString()) - + (line == null ? "" : "#" + line) - + ")"; + return "[✎](" + "https://github.com/bakdata/conquery/edit/develop/" + FilenameUtils.separatorsToUnix( + target.toString()) + (line == null ? "" : "#" + line) + ")"; } private String printType(Ctx ctx, TypeSignature type) { @@ -348,21 +353,23 @@ private String printType(Ctx ctx, TypeSignature type) { //Map if (BiMap.class.isAssignableFrom(cl)) { - return "bijective map from " - + printType(ctx.withGeneric(true), classRef.getTypeArguments().get(0)) - + " to " - + printType(ctx.withGeneric(true), classRef.getTypeArguments().get(1)); + return "bijective map from " + printType( + ctx.withGeneric(true), + classRef.getTypeArguments().get(0)) + " to " + printType( + ctx.withGeneric(true), + classRef.getTypeArguments().get(1)); } if (ClassToInstanceMap.class.isAssignableFrom(cl)) { - return "ClassToInstanceMap maps from base class " - + printType(ctx.withGeneric(true), classRef.getTypeArguments().get(0)) - + " to instances of subtypes"; + return "ClassToInstanceMap maps from base class " + printType( + ctx.withGeneric(true), + classRef.getTypeArguments().get(0)) + " to instances of subtypes"; } if (Map.class.isAssignableFrom(cl)) { - return "map from " - + printType(ctx.withGeneric(true), classRef.getTypeArguments().get(0)) - + " to " - + printType(ctx.withGeneric(true), classRef.getTypeArguments().get(1)); + return "map from " + printType( + ctx.withGeneric(true), + classRef.getTypeArguments().get(0)) + " to " + printType( + ctx.withGeneric(true), + classRef.getTypeArguments().get(1)); } @@ -376,9 +383,11 @@ private String printType(Ctx ctx, TypeSignature type) { return "[" + type.toStringWithSimpleNames() + "](" + anchor(baseTitle(cl)) + ")"; } //another contentClass - final Optional> - match = - content.values().stream().filter(p -> p.getRight().loadClass().equals(cl)).collect(MoreCollectors.toOptional()); + final Optional> match = content.values() + .stream() + .filter( + p -> p.getRight().loadClass().equals(cl)) + .collect(MoreCollectors.toOptional()); if (match.isPresent()) { return "[" + match.get().getLeft().id() + "](" + anchor(match.get().getLeft().id()) + ")"; } @@ -397,12 +406,15 @@ private String printType(Ctx ctx, TypeSignature type) { //ENUM if (Enum.class.isAssignableFrom(cl)) { - return "one of " + Arrays.stream(cl.getEnumConstants()).map(Enum.class::cast).map(Enum::name).collect(Collectors.joining(", ")); + return "one of " + Arrays.stream(cl.getEnumConstants()) + .map(Enum.class::cast) + .map(Enum::name) + .collect( + Collectors.joining(", ")); } if (Primitives.isWrapperType(cl)) { - return "`" + Primitives.unwrap(cl).getSimpleName() + "`" - + (ctx.isIdOf() ? "" : " or `null`"); + return "`" + Primitives.unwrap(cl).getSimpleName() + "`" + (ctx.isIdOf() ? "" : " or `null`"); } //default for hidden types @@ -434,14 +446,14 @@ private String printType(Ctx ctx, TypeArgument type) { } if (type.getTypeSignature() instanceof TypeVariableSignature) { final String v = type.getTypeSignature().toString(); - final TypeParameter typeParam = ctx - .getField() - .getClassInfo() - .getTypeSignature() - .getTypeParameters() - .stream() - .filter(tp -> tp.getName().equals(v)) - .collect(MoreCollectors.onlyElement()); + final TypeParameter typeParam = ctx.getField() + .getClassInfo() + .getTypeSignature() + .getTypeParameters() + .stream() + .filter( + tp -> tp.getName().equals(v)) + .collect(MoreCollectors.onlyElement()); if (typeParam.getClassBound() != null) { return printType(ctx, typeParam.getClassBound()); } @@ -466,7 +478,9 @@ private boolean isJSONSettableField(FieldInfo field) { //has @JsonCreator for (MethodInfo method : field.getClassInfo().getMethodAndConstructorInfo()) { if (method.hasAnnotation(JSON_CREATOR)) { - if (Arrays.stream(method.getParameterInfo()).anyMatch(param -> param.getName().equals(field.getName()))) { + if (Arrays.stream(method.getParameterInfo()) + .anyMatch( + param -> param.getName().equals(field.getName()))) { return true; } } diff --git a/autodoc/src/main/java/com/bakdata/conquery/handler/SimpleWriter.java b/autodoc/src/main/java/com/bakdata/conquery/handler/SimpleWriter.java index d961c135a40..a7837893e92 100644 --- a/autodoc/src/main/java/com/bakdata/conquery/handler/SimpleWriter.java +++ b/autodoc/src/main/java/com/bakdata/conquery/handler/SimpleWriter.java @@ -7,7 +7,6 @@ import java.util.stream.Stream; import com.google.common.base.Joiner; - import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -22,31 +21,32 @@ public void close() throws IOException { } public void heading(String str) throws IOException { - line("\n# "+str); + line("\n# " + str); } - + public void subHeading(String str) throws IOException { - line("\n---\n\n## "+str); + line("\n---\n\n## " + str); } - + public void subSubHeading(String str) throws IOException { - line("\n### "+str); + line("\n### " + str); } - + public void tableHeader(String... header) throws IOException { - line("\n| "+Joiner.on(" | ").join(header)+" |"); - line("| "+Stream.generate(()->"---").limit(header.length).collect(Collectors.joining(" | "))+" |"); + line("\n| " + Joiner.on(" | ").join(header) + " |"); + line("| " + Stream.generate(() -> "---").limit(header.length).collect(Collectors.joining(" | ")) + " |"); } + public void table(String... values) throws IOException { - line("| "+Joiner.on(" | ").join(values)+" | "); + line("| " + Joiner.on(" | ").join(values) + " | "); } - + public void paragraph(String str) throws IOException { - line(str+"\n"); + line(str + "\n"); } public void line(String str) throws IOException { - writer.write(str+"\n"); + writer.write(str + "\n"); log.trace(str); } } diff --git a/autodoc/src/main/java/com/bakdata/conquery/introspection/AbstractJavadocIntrospection.java b/autodoc/src/main/java/com/bakdata/conquery/introspection/AbstractJavadocIntrospection.java index 8304cc0fbbc..27a199b5564 100644 --- a/autodoc/src/main/java/com/bakdata/conquery/introspection/AbstractJavadocIntrospection.java +++ b/autodoc/src/main/java/com/bakdata/conquery/introspection/AbstractJavadocIntrospection.java @@ -40,21 +40,22 @@ private String extractExample() { } var javadoc = value.getJavadoc().get(); return javadoc.getBlockTags() - .stream() - .filter(b -> b.getTagName().equals("jsonExample")) - .findAny() - .map(JavadocBlockTag::getContent) - .map(JavadocDescription::toText) - .orElse(""); + .stream() + .filter(b -> b.getTagName().equals("jsonExample")) + .findAny() + .map( + JavadocBlockTag::getContent) + .map(JavadocDescription::toText) + .orElse(""); } @Override public String getLine() { if (value.getJavadocComment().isPresent()) { - return "L" - + value.getJavadocComment().get().getBegin().get().line - + "-L" - + value.getJavadocComment().get().getEnd().get().line; + return "L" + value.getJavadocComment().get().getBegin().get().line + "-L" + value.getJavadocComment() + .get() + .getEnd() + .get().line; } return "L" + value.getBegin().get().line; } diff --git a/autodoc/src/main/java/com/bakdata/conquery/introspection/AbstractNodeWithMemberIntrospection.java b/autodoc/src/main/java/com/bakdata/conquery/introspection/AbstractNodeWithMemberIntrospection.java index 63a6abba4c1..24f7500dabe 100644 --- a/autodoc/src/main/java/com/bakdata/conquery/introspection/AbstractNodeWithMemberIntrospection.java +++ b/autodoc/src/main/java/com/bakdata/conquery/introspection/AbstractNodeWithMemberIntrospection.java @@ -30,17 +30,18 @@ public AbstractNodeWithMemberIntrospection(File file, VALUE nodeWithJavadoc) { @Override public Introspection findMethod(MethodInfo method) { var types = Arrays.stream(method.getParameterInfo()) - .map(MethodParameterInfo::getTypeSignatureOrTypeDescriptor) - .map(this::toClass) - .toArray(Class[]::new); + .map( + MethodParameterInfo::getTypeSignatureOrTypeDescriptor) + .map(this::toClass) + .toArray(Class[]::new); return new AbstractJavadocIntrospection<>( - file, - value - .getMethodsByParameterTypes(types) - .stream() - .filter(md -> md.getNameAsString().equals(method.getName())) - .collect(MoreCollectors.onlyElement()) + file, + value.getMethodsByParameterTypes(types) + .stream() + .filter( + md -> md.getNameAsString().equals(method.getName())) + .collect(MoreCollectors.onlyElement()) ); } @@ -63,8 +64,7 @@ public Introspection findInnerType(String simpleName) { if (!node.getNameAsString().equals(simpleName)) { continue; } - } - else { + } else { continue; } if (decl instanceof EnumDeclaration enumDeclaration) { @@ -80,11 +80,9 @@ public Introspection findInnerType(String simpleName) { private Class toClass(TypeSignature sig) { if (sig instanceof BaseTypeSignature) { return ((BaseTypeSignature) sig).getType(); - } - else if (sig instanceof ArrayTypeSignature) { + } else if (sig instanceof ArrayTypeSignature) { return ((ArrayTypeSignature) sig).loadClass(); - } - else if (sig instanceof ClassRefTypeSignature) { + } else if (sig instanceof ClassRefTypeSignature) { return ((ClassRefTypeSignature) sig).loadClass(); } throw new IllegalStateException("Can't find class for signature " + sig); diff --git a/autodoc/src/main/java/com/bakdata/conquery/introspection/EnumIntrospection.java b/autodoc/src/main/java/com/bakdata/conquery/introspection/EnumIntrospection.java index d6f271b97b9..00df0fe815a 100644 --- a/autodoc/src/main/java/com/bakdata/conquery/introspection/EnumIntrospection.java +++ b/autodoc/src/main/java/com/bakdata/conquery/introspection/EnumIntrospection.java @@ -17,9 +17,14 @@ public EnumIntrospection(File file, EnumDeclaration enumDeclaration) { protected String extractDescription() { String description = super.extractDescription(); - String values = value.getEntries().stream() - .map(e -> "`%s`: %s".formatted(e.getNameAsString(), e.getJavadoc().map(Javadoc::toText).orElse("No description available"))) - .collect(Collectors.joining("\n- ", "\nValues:\n- ", "")); + String values = value.getEntries() + .stream() + .map( + e -> "`%s`: %s".formatted( + e.getNameAsString(), + e.getJavadoc().map(Javadoc::toText).orElse("No description available"))) + .collect( + Collectors.joining("\n- ", "\nValues:\n- ", "")); description += values; return description; } diff --git a/autodoc/src/main/java/com/bakdata/conquery/introspection/Introspection.java b/autodoc/src/main/java/com/bakdata/conquery/introspection/Introspection.java index b5c9191b0ad..9485e1e318b 100644 --- a/autodoc/src/main/java/com/bakdata/conquery/introspection/Introspection.java +++ b/autodoc/src/main/java/com/bakdata/conquery/introspection/Introspection.java @@ -13,13 +13,15 @@ public interface Introspection { String getDescription(); + String getExample(); + File getFile(); static Introspection from(File root, ClassInfo cl) { - File f = new File(root, "backend/src/main/java/"+cl.getName().replace('.', '/')+".java"); + File f = new File(root, "backend/src/main/java/" + cl.getName().replace('.', '/') + ".java"); try { - if(cl.isInnerClass()) { + if (cl.isInnerClass()) { Introspection from = from(root, cl.getOuterClasses().get(cl.getOuterClasses().size() - 1)); if (from instanceof SimpleIntrospection simpleIntrospection) { return simpleIntrospection; @@ -27,13 +29,12 @@ static Introspection from(File root, ClassInfo cl) { if (from instanceof AbstractNodeWithMemberIntrospection nodeWithMemberIntrospection) { return nodeWithMemberIntrospection.findInnerType(cl.getSimpleName()); - } - else { + } else { return new SimpleIntrospection(f); } } - - + + CompilationUnit cu = StaticJavaParser.parse(f); TypeDeclaration typeDeclaration = cu.getPrimaryType().get(); @@ -42,12 +43,19 @@ static Introspection from(File root, ClassInfo cl) { return new EnumIntrospection(f, enumDeclaration); } return new AbstractNodeWithMemberIntrospection<>(f, typeDeclaration); - } catch(Exception e) { - LoggerFactory.getLogger(Introspection.class).warn("Could not create compilation unit for {}", cl.getName(), e); + } catch (Exception e) { + LoggerFactory.getLogger(Introspection.class) + .warn( + "Could not create compilation unit for {}", + cl.getName(), + e); return new SimpleIntrospection(f); } } + String getLine(); + Introspection findField(FieldInfo field); + Introspection findMethod(MethodInfo method); } diff --git a/autodoc/src/main/java/com/bakdata/conquery/introspection/SimpleIntrospection.java b/autodoc/src/main/java/com/bakdata/conquery/introspection/SimpleIntrospection.java index f6214307e5f..170d7ee0fdd 100644 --- a/autodoc/src/main/java/com/bakdata/conquery/introspection/SimpleIntrospection.java +++ b/autodoc/src/main/java/com/bakdata/conquery/introspection/SimpleIntrospection.java @@ -10,7 +10,7 @@ public class SimpleIntrospection implements Introspection { private final File file; - + @Override public String getDescription() { return ""; diff --git a/autodoc/src/main/java/com/bakdata/conquery/model/Base.java b/autodoc/src/main/java/com/bakdata/conquery/model/Base.java index 8c359e7d53c..94c48e4ee02 100644 --- a/autodoc/src/main/java/com/bakdata/conquery/model/Base.java +++ b/autodoc/src/main/java/com/bakdata/conquery/model/Base.java @@ -4,8 +4,9 @@ import lombok.Getter; import lombok.ToString; -@ToString(of="baseClass") -@AllArgsConstructor @Getter +@ToString(of = "baseClass") +@AllArgsConstructor +@Getter public class Base { private Class baseClass; private String description; diff --git a/autodoc/src/main/java/com/bakdata/conquery/model/Group.java b/autodoc/src/main/java/com/bakdata/conquery/model/Group.java index 62326802f08..0251fea7630 100644 --- a/autodoc/src/main/java/com/bakdata/conquery/model/Group.java +++ b/autodoc/src/main/java/com/bakdata/conquery/model/Group.java @@ -7,7 +7,8 @@ import lombok.Getter; import lombok.Singular; -@Getter @Builder +@Getter +@Builder public class Group { private String name; @@ -22,4 +23,4 @@ public class Group { private List> hides; @Singular private List> markerInterfaces; -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/Conquery.java b/backend/src/main/java/com/bakdata/conquery/Conquery.java index 38531ae726f..c1e9c37910d 100644 --- a/backend/src/main/java/com/bakdata/conquery/Conquery.java +++ b/backend/src/main/java/com/bakdata/conquery/Conquery.java @@ -1,7 +1,7 @@ package com.bakdata.conquery; -import java.time.Clock; import jakarta.validation.Validator; +import java.time.Clock; import ch.qos.logback.classic.Level; import com.bakdata.conquery.commands.DistributedStandaloneCommand; @@ -78,8 +78,10 @@ public void initialize(Bootstrap bootstrap) { @Override public void initialize(Bootstrap bootstrap) { // Allow overriding of config from environment variables. - bootstrap.setConfigurationSourceProvider(new SubstitutingSourceProvider( - bootstrap.getConfigurationSourceProvider(), StringSubstitutor.createInterpolator())); + bootstrap.setConfigurationSourceProvider( + new SubstitutingSourceProvider( + bootstrap.getConfigurationSourceProvider(), + StringSubstitutor.createInterpolator())); } @Override @@ -107,8 +109,8 @@ protected Level bootstrapLogLevel() { @Override public void run(ConqueryConfig configuration, Environment environment) throws Exception { - ManagerProvider provider = configuration.getSqlConnectorConfig().isEnabled() ? - new LocalManagerProvider(getQueryClock()) : new ClusterManagerProvider(); + ManagerProvider provider = configuration.getSqlConnectorConfig().isEnabled() ? new LocalManagerProvider( + getQueryClock()) : new ClusterManagerProvider(); Manager manager = provider.provideManager(configuration, environment); ManagerNode managerNode = new ManagerNode(); diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/AdditionalMediaTypes.java b/backend/src/main/java/com/bakdata/conquery/apiv1/AdditionalMediaTypes.java index d592edd5a84..02439c59ee2 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/AdditionalMediaTypes.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/AdditionalMediaTypes.java @@ -6,7 +6,7 @@ public interface AdditionalMediaTypes { static final String CSV = "text/csv; charset=utf-8"; static final String EXCEL = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; - + // As proposed in https://issues.apache.org/jira/browse/ARROW-7396 static final String ARROW_STREAM = "application/vnd.apache.arrow.stream"; static final String ARROW_FILE = "application/vnd.apache.arrow.file"; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/ApiV1.java b/backend/src/main/java/com/bakdata/conquery/apiv1/ApiV1.java index 46cc0f38b93..2fa2ca9f40d 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/ApiV1.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/ApiV1.java @@ -1,87 +1,87 @@ -package com.bakdata.conquery.apiv1; - -import com.bakdata.conquery.commands.ManagerNode; -import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.io.jersey.IdPathParamConverterProvider; -import com.bakdata.conquery.io.jetty.CORSPreflightRequestFilter; -import com.bakdata.conquery.io.jetty.CORSResponseFilter; -import com.bakdata.conquery.io.result.ResultRender.ResultRendererProvider; -import com.bakdata.conquery.metrics.ActiveUsersFilter; -import com.bakdata.conquery.models.auth.basic.JWTokenHandler; -import com.bakdata.conquery.models.auth.web.AuthFilter; -import com.bakdata.conquery.models.forms.frontendconfiguration.FormConfigProcessor; -import com.bakdata.conquery.models.forms.frontendconfiguration.FormProcessor; -import com.bakdata.conquery.resources.ResourcesProvider; -import com.bakdata.conquery.resources.api.ConceptResource; -import com.bakdata.conquery.resources.api.ConceptsProcessor; -import com.bakdata.conquery.resources.api.ConfigResource; -import com.bakdata.conquery.resources.api.DatasetFormConfigResource; -import com.bakdata.conquery.resources.api.DatasetFormResource; -import com.bakdata.conquery.resources.api.DatasetQueryResource; -import com.bakdata.conquery.resources.api.DatasetResource; -import com.bakdata.conquery.resources.api.DatasetsResource; -import com.bakdata.conquery.resources.api.FilterResource; -import com.bakdata.conquery.resources.api.FormConfigResource; -import com.bakdata.conquery.resources.api.MeResource; -import com.bakdata.conquery.resources.api.QueryResource; -import io.dropwizard.jersey.setup.JerseyEnvironment; -import org.glassfish.jersey.internal.inject.AbstractBinder; -import org.glassfish.jersey.server.ResourceConfig; - -@CPSType(base = ResourcesProvider.class, id = "ApiV1") -public class ApiV1 extends ResourceConfig implements ResourcesProvider { - - @Override - public void registerResources(ManagerNode manager) { - - final JerseyEnvironment jersey = manager.getEnvironment().jersey(); - // TODO this does not work, if we really want to do api versioning - jersey.setUrlPattern("/api"); - - // Inject Processors - jersey.register(new AbstractBinder() { - @Override - protected void configure() { - bindAsContract(QueryProcessor.class); - bindAsContract(ConceptsProcessor.class); - bindAsContract(MeProcessor.class); - bindAsContract(FormConfigProcessor.class); - bindAsContract(FormProcessor.class); - } - }); - - jersey.register(CORSPreflightRequestFilter.class); - jersey.register(CORSResponseFilter.class); - - jersey.register(ActiveUsersFilter.class); - - - /* - * Register the authentication filter which protects all resources registered in this servlet. - * We use the same instance of the filter for the api servlet and the admin servlet to have a single - * point for authentication. - */ - jersey.register(AuthFilter.class); - AuthFilter.registerTokenExtractor(JWTokenHandler.JWTokenExtractor.class, jersey.getResourceConfig()); - - jersey.register(IdPathParamConverterProvider.class); - - jersey.register(QueryResource.class); - jersey.register(DatasetQueryResource.class); - jersey.register(DatasetFormConfigResource.class); - jersey.register(ConfigResource.class); - jersey.register(FormConfigResource.class); - jersey.register(DatasetFormResource.class); - - jersey.register(DatasetsResource.class); - jersey.register(ConceptResource.class); - jersey.register(DatasetResource.class); - jersey.register(FilterResource.class); - jersey.register(MeResource.class); - - for (ResultRendererProvider resultProvider : manager.getConfig().getResultProviders()) { - resultProvider.registerResultResource(jersey.getResourceConfig(), manager); - } - - } -} +package com.bakdata.conquery.apiv1; + +import com.bakdata.conquery.commands.ManagerNode; +import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.io.jersey.IdPathParamConverterProvider; +import com.bakdata.conquery.io.jetty.CORSPreflightRequestFilter; +import com.bakdata.conquery.io.jetty.CORSResponseFilter; +import com.bakdata.conquery.io.result.ResultRender.ResultRendererProvider; +import com.bakdata.conquery.metrics.ActiveUsersFilter; +import com.bakdata.conquery.models.auth.basic.JWTokenHandler; +import com.bakdata.conquery.models.auth.web.AuthFilter; +import com.bakdata.conquery.models.forms.frontendconfiguration.FormConfigProcessor; +import com.bakdata.conquery.models.forms.frontendconfiguration.FormProcessor; +import com.bakdata.conquery.resources.ResourcesProvider; +import com.bakdata.conquery.resources.api.ConceptResource; +import com.bakdata.conquery.resources.api.ConceptsProcessor; +import com.bakdata.conquery.resources.api.ConfigResource; +import com.bakdata.conquery.resources.api.DatasetFormConfigResource; +import com.bakdata.conquery.resources.api.DatasetFormResource; +import com.bakdata.conquery.resources.api.DatasetQueryResource; +import com.bakdata.conquery.resources.api.DatasetResource; +import com.bakdata.conquery.resources.api.DatasetsResource; +import com.bakdata.conquery.resources.api.FilterResource; +import com.bakdata.conquery.resources.api.FormConfigResource; +import com.bakdata.conquery.resources.api.MeResource; +import com.bakdata.conquery.resources.api.QueryResource; +import io.dropwizard.jersey.setup.JerseyEnvironment; +import org.glassfish.jersey.internal.inject.AbstractBinder; +import org.glassfish.jersey.server.ResourceConfig; + +@CPSType(base = ResourcesProvider.class, id = "ApiV1") +public class ApiV1 extends ResourceConfig implements ResourcesProvider { + + @Override + public void registerResources(ManagerNode manager) { + + final JerseyEnvironment jersey = manager.getEnvironment().jersey(); + // TODO this does not work, if we really want to do api versioning + jersey.setUrlPattern("/api"); + + // Inject Processors + jersey.register(new AbstractBinder() { + @Override + protected void configure() { + bindAsContract(QueryProcessor.class); + bindAsContract(ConceptsProcessor.class); + bindAsContract(MeProcessor.class); + bindAsContract(FormConfigProcessor.class); + bindAsContract(FormProcessor.class); + } + }); + + jersey.register(CORSPreflightRequestFilter.class); + jersey.register(CORSResponseFilter.class); + + jersey.register(ActiveUsersFilter.class); + + + /* + * Register the authentication filter which protects all resources registered in this servlet. + * We use the same instance of the filter for the api servlet and the admin servlet to have a single + * point for authentication. + */ + jersey.register(AuthFilter.class); + AuthFilter.registerTokenExtractor(JWTokenHandler.JWTokenExtractor.class, jersey.getResourceConfig()); + + jersey.register(IdPathParamConverterProvider.class); + + jersey.register(QueryResource.class); + jersey.register(DatasetQueryResource.class); + jersey.register(DatasetFormConfigResource.class); + jersey.register(ConfigResource.class); + jersey.register(FormConfigResource.class); + jersey.register(DatasetFormResource.class); + + jersey.register(DatasetsResource.class); + jersey.register(ConceptResource.class); + jersey.register(DatasetResource.class); + jersey.register(FilterResource.class); + jersey.register(MeResource.class); + + for (ResultRendererProvider resultProvider : manager.getConfig().getResultProviders()) { + resultProvider.registerResultResource(jersey.getResourceConfig(), manager); + } + + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java b/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java index 92b3182c5b4..71c7cb97f45 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java @@ -1,8 +1,8 @@ package com.bakdata.conquery.apiv1; -import java.net.URI; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; +import java.net.URI; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.config.ConqueryConfig; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/FormConfigPatch.java b/backend/src/main/java/com/bakdata/conquery/apiv1/FormConfigPatch.java index 0c6862a753e..fca768e0c07 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/FormConfigPatch.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/FormConfigPatch.java @@ -1,39 +1,42 @@ -package com.bakdata.conquery.apiv1; - -import java.util.function.Consumer; - -import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.models.auth.entities.Subject; -import com.bakdata.conquery.models.auth.permissions.Ability; -import com.bakdata.conquery.models.forms.configs.FormConfig; -import com.bakdata.conquery.util.QueryUtils; -import com.fasterxml.jackson.databind.JsonNode; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; -import lombok.experimental.SuperBuilder; - -/** - * Specific class to also patch the values stored in a {@link FormConfig}. - */ -@SuperBuilder -@EqualsAndHashCode(callSuper = true) -@Data -@NoArgsConstructor -public class FormConfigPatch extends MetaDataPatch { - private JsonNode values; - - public void applyTo(FormConfig instance, MetaStorage storage, Subject subject){ - chain(QueryUtils.getNoOpEntryPoint(), storage, subject, instance) - .accept(this); - } - - protected Consumer chain(Consumer patchConsumerChain, MetaStorage storage, Subject subject, FormConfig instance) { - patchConsumerChain = super.buildChain(patchConsumerChain, storage, subject, instance); - - if(getValues() != null && subject.isPermitted(instance,Ability.MODIFY)) { - patchConsumerChain = patchConsumerChain.andThen(instance.valueSetter()); - } - return patchConsumerChain; - } -} +package com.bakdata.conquery.apiv1; + +import java.util.function.Consumer; + +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.models.auth.entities.Subject; +import com.bakdata.conquery.models.auth.permissions.Ability; +import com.bakdata.conquery.models.forms.configs.FormConfig; +import com.bakdata.conquery.util.QueryUtils; +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +/** + * Specific class to also patch the values stored in a {@link FormConfig}. + */ +@SuperBuilder +@EqualsAndHashCode(callSuper = true) +@Data +@NoArgsConstructor +public class FormConfigPatch extends MetaDataPatch { + private JsonNode values; + + public void applyTo(FormConfig instance, MetaStorage storage, Subject subject) { + chain(QueryUtils.getNoOpEntryPoint(), storage, subject, instance).accept(this); + } + + protected Consumer chain( + Consumer patchConsumerChain, + MetaStorage storage, + Subject subject, + FormConfig instance) { + patchConsumerChain = super.buildChain(patchConsumerChain, storage, subject, instance); + + if (getValues() != null && subject.isPermitted(instance, Ability.MODIFY)) { + patchConsumerChain = patchConsumerChain.andThen(instance.valueSetter()); + } + return patchConsumerChain; + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/KeyValue.java b/backend/src/main/java/com/bakdata/conquery/apiv1/KeyValue.java index 59a84d18735..31cbb2e560c 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/KeyValue.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/KeyValue.java @@ -11,8 +11,8 @@ public class KeyValue implements Serializable { private static final long serialVersionUID = 1L; - + private String key; private String value; - + } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/MeProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/MeProcessor.java index a7d7494735d..805cf29b1e5 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/MeProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/MeProcessor.java @@ -1,10 +1,10 @@ package com.bakdata.conquery.apiv1; +import jakarta.inject.Inject; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import jakarta.inject.Inject; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.AuthorizationHelper; @@ -52,26 +52,29 @@ public FrontendMeInformation getUserInformation(@NonNull User user) { // User can use the dataset and can possibly upload ids for resolving datasetAblilites.put( - dataset, - new FrontendDatasetAbility( - user.isPermitted(dataset, Ability.PRESERVE_ID), - user.isPermitted(dataset, Ability.ENTITY_PREVIEW) && user.isPermitted(dataset, Ability.PRESERVE_ID), - user.isPermitted(dataset, Ability.QUERY_PREVIEW) - ) + dataset, + new FrontendDatasetAbility( + user.isPermitted(dataset, Ability.PRESERVE_ID), + user.isPermitted(dataset, Ability.ENTITY_PREVIEW) && user.isPermitted(dataset, Ability.PRESERVE_ID), + user.isPermitted(dataset, Ability.QUERY_PREVIEW) + ) ); } // Build user information return FrontendMeInformation.builder() - .userName(user.getLabel()) - .hideLogoutButton(!user.isDisplayLogout()) - .groups( - AuthorizationHelper.getGroupsOf(user, storage) - .stream() - .map(g -> new IdLabel(g.getId(), g.getLabel())) - .collect(Collectors.toList())) - .datasetAbilities(datasetAblilites) - .build(); + .userName(user.getLabel()) + .hideLogoutButton( + !user.isDisplayLogout()) + .groups( + AuthorizationHelper.getGroupsOf(user, storage) + .stream() + .map( + g -> new IdLabel(g.getId(), g.getLabel())) + .collect(Collectors.toList())) + .datasetAbilities( + datasetAblilites) + .build(); } /** diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/MetaDataPatch.java b/backend/src/main/java/com/bakdata/conquery/apiv1/MetaDataPatch.java index 26b1df61592..7d3069a01a4 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/MetaDataPatch.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/MetaDataPatch.java @@ -1,69 +1,69 @@ -package com.bakdata.conquery.apiv1; - -import java.util.List; -import java.util.function.Consumer; - -import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.models.auth.entities.Group; -import com.bakdata.conquery.models.auth.entities.Subject; -import com.bakdata.conquery.models.auth.permissions.Ability; -import com.bakdata.conquery.models.auth.permissions.Authorized; -import com.bakdata.conquery.models.execution.Labelable; -import com.bakdata.conquery.models.execution.Owned; -import com.bakdata.conquery.models.execution.Shareable; -import com.bakdata.conquery.models.execution.Shareable.ShareInformation; -import com.bakdata.conquery.models.execution.Taggable; -import com.bakdata.conquery.models.identifiable.Identifiable; -import com.bakdata.conquery.models.identifiable.ids.Id; -import com.bakdata.conquery.models.identifiable.ids.specific.GroupId; -import com.bakdata.conquery.util.QueryUtils; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; -import lombok.experimental.SuperBuilder; -import org.apache.shiro.authz.Permission; - -@Data -@SuperBuilder -@EqualsAndHashCode(callSuper = false) -@NoArgsConstructor -public class MetaDataPatch implements Taggable, Labelable, ShareInformation { - - private String[] tags; - private String label; - private List groups; - - /** - * Patches the given {@link Identifiable} by checking if the subject holds the necessary Permission for that operation. - * Hence the patched instance must have a corresponding {@link Permission}-type. - * Tagging and Labeling only alters the state of the instance while sharing also alters the state of {@link Group}s. - * - * @param instance The instance to patch - * @param storage Storage that persists the instance and also auth information. - * @param subject The subject on whose behalf the patch is executed - * @param Type of the instance that is patched - */ - public , ?> & Owned & Authorized> void applyTo( - INST instance, - MetaStorage storage, - Subject subject) { - buildChain(QueryUtils.getNoOpEntryPoint(), storage, subject, instance).accept(this); - } - - protected , ?> & Owned & Authorized> Consumer buildChain( - Consumer patchConsumerChain, - MetaStorage storage, - Subject subject, - INST instance) { - if (getTags() != null && subject.isPermitted(instance, Ability.TAG)) { - patchConsumerChain = patchConsumerChain.andThen(instance.tagger()); - } - if (getLabel() != null && subject.isPermitted(instance, Ability.LABEL)) { - patchConsumerChain = patchConsumerChain.andThen(instance.labeler()); - } - if (getGroups() != null && subject.isPermitted(instance, Ability.SHARE)) { - patchConsumerChain = patchConsumerChain.andThen(instance.sharer(storage, subject)); - } - return patchConsumerChain; - } -} +package com.bakdata.conquery.apiv1; + +import java.util.List; +import java.util.function.Consumer; + +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.models.auth.entities.Group; +import com.bakdata.conquery.models.auth.entities.Subject; +import com.bakdata.conquery.models.auth.permissions.Ability; +import com.bakdata.conquery.models.auth.permissions.Authorized; +import com.bakdata.conquery.models.execution.Labelable; +import com.bakdata.conquery.models.execution.Owned; +import com.bakdata.conquery.models.execution.Shareable; +import com.bakdata.conquery.models.execution.Shareable.ShareInformation; +import com.bakdata.conquery.models.execution.Taggable; +import com.bakdata.conquery.models.identifiable.Identifiable; +import com.bakdata.conquery.models.identifiable.ids.Id; +import com.bakdata.conquery.models.identifiable.ids.specific.GroupId; +import com.bakdata.conquery.util.QueryUtils; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; +import org.apache.shiro.authz.Permission; + +@Data +@SuperBuilder +@EqualsAndHashCode(callSuper = false) +@NoArgsConstructor +public class MetaDataPatch implements Taggable, Labelable, ShareInformation { + + private String[] tags; + private String label; + private List groups; + + /** + * Patches the given {@link Identifiable} by checking if the subject holds the necessary Permission for that operation. + * Hence the patched instance must have a corresponding {@link Permission}-type. + * Tagging and Labeling only alters the state of the instance while sharing also alters the state of {@link Group}s. + * + * @param instance The instance to patch + * @param storage Storage that persists the instance and also auth information. + * @param subject The subject on whose behalf the patch is executed + * @param Type of the instance that is patched + */ + public , ?> & Owned & Authorized> void applyTo( + INST instance, + MetaStorage storage, + Subject subject) { + buildChain(QueryUtils.getNoOpEntryPoint(), storage, subject, instance).accept(this); + } + + protected , ?> & Owned & Authorized> Consumer buildChain( + Consumer patchConsumerChain, + MetaStorage storage, + Subject subject, + INST instance) { + if (getTags() != null && subject.isPermitted(instance, Ability.TAG)) { + patchConsumerChain = patchConsumerChain.andThen(instance.tagger()); + } + if (getLabel() != null && subject.isPermitted(instance, Ability.LABEL)) { + patchConsumerChain = patchConsumerChain.andThen(instance.labeler()); + } + if (getGroups() != null && subject.isPermitted(instance, Ability.SHARE)) { + patchConsumerChain = patchConsumerChain.andThen(instance.sharer(storage, subject)); + } + return patchConsumerChain; + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index 485141efea2..a012232791a 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -1,5 +1,11 @@ package com.bakdata.conquery.apiv1; +import jakarta.inject.Inject; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.validation.Validator; +import jakarta.ws.rs.BadRequestException; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.UriBuilder; import java.text.NumberFormat; import java.time.LocalDate; import java.util.*; @@ -57,12 +63,6 @@ import com.bakdata.conquery.util.QueryUtils; import com.bakdata.conquery.util.QueryUtils.NamespacedIdentifiableCollector; import com.bakdata.conquery.util.io.IdColumnUtil; -import jakarta.inject.Inject; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.validation.Validator; -import jakarta.ws.rs.BadRequestException; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.core.UriBuilder; import lombok.AllArgsConstructor; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -111,68 +111,81 @@ private static boolean canFrontendRender(ManagedExecution q) { * @return The modified status */ public static List getResultAssets( - List renderer, ManagedExecution exec, - UriBuilder uriBuilder, boolean allProviders) { - - return renderer.stream() - .map(rendererProvider -> { - try { - return rendererProvider.generateResultURLs(exec, uriBuilder.clone(), allProviders); - } catch (Exception e) { - log.error("Cannot generate result urls for execution '{}' with provider {}", exec.getId(), rendererProvider, e); - return null; - } - }) - .filter(Objects::nonNull) - .flatMap(Collection::stream) - .toList(); + List renderer, + ManagedExecution exec, + UriBuilder uriBuilder, + boolean allProviders) { + + return renderer.stream().map(rendererProvider -> { + try { + return rendererProvider.generateResultURLs(exec, uriBuilder.clone(), allProviders); + } catch (Exception e) { + log.error( + "Cannot generate result urls for execution '{}' with provider {}", + exec.getId(), + rendererProvider, + e); + return null; + } + }).filter(Objects::nonNull).flatMap(Collection::stream).toList(); } - public List getAllQueries(DatasetId dataset, HttpServletRequest req, Subject subject, boolean allProviders) { + public List getAllQueries( + DatasetId dataset, + HttpServletRequest req, + Subject subject, + boolean allProviders) { try (Stream allQueries = storage.getAllExecutionIds()) { - return getQueriesFiltered(dataset, RequestAwareUriBuilder.fromRequest(req), subject, allQueries, allProviders).toList(); + return getQueriesFiltered( + dataset, + RequestAwareUriBuilder.fromRequest(req), + subject, + allQueries, + allProviders).toList(); } } public Stream getQueriesFiltered( - DatasetId datasetId, - UriBuilder uriBuilder, - Subject subject, - Stream allQueries, - boolean allProviders) { + DatasetId datasetId, + UriBuilder uriBuilder, + Subject subject, + Stream allQueries, + boolean allProviders) { return allQueries - // Checks that are possible on the execution id alone (no cache load necessary) - .filter(id -> id.getDataset().equals(datasetId)) - .filter(id -> { - ExecutionManager executionManager = datasetRegistry.get(id.getDataset()).getExecutionManager(); - final ExecutionState state = executionManager.getState(id); - return state == ExecutionState.NEW || state == ExecutionState.DONE; - }) - // Resolve - .map(Id::get) - .filter(Objects::nonNull) // Cautionary - // Checks that are only possible on the execution object (cache might need to fetch some data) - .filter(QueryProcessor::canFrontendRender) - // Ownership check only works on the object not on the id :/ - .filter(exec -> subject.isPermitted(exec, Ability.READ)) - .filter(Predicate.not(ManagedExecution::isSystem)) - - .map(mq -> { - try { - final OverviewExecutionStatus status = mq.buildStatusOverview(subject); - - if (mq.isReadyToDownload()) { - status.setResultUrls(getResultAssets(config.getResultProviders(), mq, uriBuilder, allProviders)); - } - return status; - } catch (Exception e) { - log.error("FAILED building status for {}", mq, e); + // Checks that are possible on the execution id alone (no cache load necessary) + .filter(id -> id.getDataset().equals(datasetId)) + .filter(id -> { + ExecutionManager executionManager = datasetRegistry.get(id.getDataset()).getExecutionManager(); + final ExecutionState state = executionManager.getState(id); + return state == ExecutionState.NEW || state == ExecutionState.DONE; + }) + // Resolve + .map(Id::get) + .filter(Objects::nonNull) // Cautionary + // Checks that are only possible on the execution object (cache might need to fetch some data) + .filter(QueryProcessor::canFrontendRender) + // Ownership check only works on the object not on the id :/ + .filter(exec -> subject.isPermitted(exec, Ability.READ)) + .filter( + Predicate.not(ManagedExecution::isSystem)) + + .map(mq -> { + try { + final OverviewExecutionStatus status = mq.buildStatusOverview(subject); + + if (mq.isReadyToDownload()) { + status.setResultUrls( + getResultAssets(config.getResultProviders(), mq, uriBuilder, allProviders)); } - return null; - }) - .filter(Objects::nonNull); + return status; + } catch (Exception e) { + log.error("FAILED building status for {}", mq, e); + } + return null; + }) + .filter(Objects::nonNull); } /** @@ -226,8 +239,9 @@ public void patchQuery(Subject subject, ManagedExecutionId executionId, MetaData } final MetaDataPatch sharePatch = MetaDataPatch.builder() - .groups(new ArrayList<>(groupsToShareWith)) - .build(); + .groups( + new ArrayList<>(groupsToShareWith)) + .build(); patchQuery(subject, subExecutionId, sharePatch); } @@ -257,14 +271,19 @@ public void deleteQuery(Subject subject, ManagedExecutionId executionId) { log.info("User[{}] deleted Query[{}]", subject.getId(), executionId); datasetRegistry.get(executionId.getDataset()) - .getExecutionManager() // Don't go over execution#getExecutionManager() as that's only set when query is initialized - .clearQueryResults(executionId); + .getExecutionManager() // Don't go over execution#getExecutionManager() as that's only set when query is initialized + .clearQueryResults(executionId); storage.removeExecution(executionId); } - public FullExecutionStatus getQueryFullStatus(ManagedExecutionId queryId, Subject subject, UriBuilder url, Boolean allProviders, boolean await) { + public FullExecutionStatus getQueryFullStatus( + ManagedExecutionId queryId, + Subject subject, + UriBuilder url, + Boolean allProviders, + boolean await) { final Namespace namespace = datasetRegistry.get(queryId.getDataset()); if (await) { @@ -289,30 +308,39 @@ public FullExecutionStatus getQueryFullStatus(ManagedExecutionId queryId, Subjec public ExternalUploadResult uploadEntities(Subject subject, DatasetId dataset, ExternalUpload upload) { final Namespace namespace = datasetRegistry.get(dataset); - final EntityResolver.ResolveStatistic statistic = namespace.getEntityResolver().resolveEntities( + final EntityResolver.ResolveStatistic statistic = namespace.getEntityResolver() + .resolveEntities( upload.getValues(), upload.getFormat(), namespace.getStorage().getIdMapping(), config.getIdColumns(), config.getLocale().getDateReader(), upload.isOneRowPerEntity() - ); + ); // Resolving nothing is a problem thus we fail. if (statistic.getResolved().isEmpty()) { - throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST) - .entity(new ExternalUploadResult(null, 0, statistic.getUnresolvedId(), statistic.getUnreadableDate())) + throw new BadRequestException( + Response.status(Response.Status.BAD_REQUEST) + .entity( + new ExternalUploadResult( + null, + 0, + statistic.getUnresolvedId(), + statistic.getUnreadableDate())) .build()); } - final ConceptQuery query = new ConceptQuery(new CQExternal(upload.getFormat(), upload.getValues(), upload.isOneRowPerEntity())); + final ConceptQuery query = new ConceptQuery( + new CQExternal(upload.getFormat(), upload.getValues(), upload.isOneRowPerEntity())); // We only create the Query, really no need to execute it as it's only useful for composition. - final ManagedQuery - execution = - ((ManagedQuery) namespace - .getExecutionManager() - .createExecution(query, subject.getId(), namespace, false)); + final ManagedQuery execution = ((ManagedQuery) namespace.getExecutionManager() + .createExecution( + query, + subject.getId(), + namespace, + false)); if (upload.getLabel() != null) { execution.setLabel(upload.getLabel()); @@ -320,25 +348,35 @@ public ExternalUploadResult uploadEntities(Subject subject, DatasetId dataset, E execution.initExecutable(); - return new ExternalUploadResult(execution.getId(), statistic.getResolved().size(), statistic.getUnresolvedId(), statistic.getUnreadableDate()); + return new ExternalUploadResult( + execution.getId(), + statistic.getResolved().size(), + statistic.getUnresolvedId(), + statistic.getUnreadableDate()); } /** * Create and submit {@link EntityPreviewForm} for subject on to extract sources for entity, and extract some additional infos to be used as infocard. */ public FullExecutionStatus getSingleEntityExport( - Subject subject, - UriBuilder uriBuilder, - String idKind, - String entity, - List sources, - DatasetId dataset, - Range dateRange) { + Subject subject, + UriBuilder uriBuilder, + String idKind, + String entity, + List sources, + DatasetId dataset, + Range dateRange) { final Namespace namespace = datasetRegistry.get(dataset); final PreviewConfig previewConfig = namespace.getPreviewConfig(); - final EntityPreviewForm form = - EntityPreviewForm.create(entity, idKind, dateRange, sources, previewConfig.getSelects(), previewConfig.getTimeStratifiedSelects(), datasetRegistry); + final EntityPreviewForm form = EntityPreviewForm.create( + entity, + idKind, + dateRange, + sources, + previewConfig.getSelects(), + previewConfig.getTimeStratifiedSelects(), + datasetRegistry); // Validate our own form because we provide it directly to the processor, which does not validate. ValidatorHelper.failOnError(log, validator.validate(form)); @@ -370,7 +408,11 @@ public FullExecutionStatus getSingleEntityExport( * Creates a query for all datasets, then submits it for execution on the * intended dataset. */ - public ManagedExecution postQuery(DatasetId dataset, QueryDescription queryContent, Subject subject, boolean system) { + public ManagedExecution postQuery( + DatasetId dataset, + QueryDescription queryContent, + Subject subject, + boolean system) { log.info("Query posted on Dataset[{}] by User[{{}].", dataset, subject.getId()); @@ -386,8 +428,12 @@ public ManagedExecution postQuery(DatasetId dataset, QueryDescription queryConte final NamespacedIdentifiableCollector namespacedIdentifiableCollector = new NamespacedIdentifiableCollector(); visitors.add(namespacedIdentifiableCollector); - final String primaryGroupName = AuthorizationHelper.getPrimaryGroup(subject, storage).map(Group::getName).orElse("none"); - final ExecutionMetrics.QueryMetricsReporter queryMetricsReporter = new ExecutionMetrics.QueryMetricsReporter(primaryGroupName); + final String primaryGroupName = AuthorizationHelper.getPrimaryGroup(subject, storage) + .map( + Group::getName) + .orElse("none"); + final ExecutionMetrics.QueryMetricsReporter queryMetricsReporter = new ExecutionMetrics.QueryMetricsReporter( + primaryGroupName); visitors.add(queryMetricsReporter); @@ -417,9 +463,8 @@ public ManagedExecution postQuery(DatasetId dataset, QueryDescription queryConte { final Optional executionId = onlyReusingChecker.getOnlyReused(); - final Optional - execution = - executionId.map(id -> tryReuse(queryContent, id, namespace, executionManager, subject.getUser())); + final Optional execution = executionId.map( + id -> tryReuse(queryContent, id, namespace, executionManager, subject.getUser())); if (execution.isPresent()) { return execution.get(); @@ -433,7 +478,12 @@ public ManagedExecution postQuery(DatasetId dataset, QueryDescription queryConte /** * Determine if the submitted query does reuse ONLY another query and restart that instead of creating another one. */ - private ManagedExecution tryReuse(QueryDescription query, ManagedExecutionId executionId, Namespace namespace, ExecutionManager executionManager, User user) { + private ManagedExecution tryReuse( + QueryDescription query, + ManagedExecutionId executionId, + Namespace namespace, + ExecutionManager executionManager, + User user) { ManagedExecution execution = storage.getExecution(executionId); @@ -449,7 +499,8 @@ private ManagedExecution tryReuse(QueryDescription query, ManagedExecutionId exe // If SecondaryIds differ from selected and prior, we cannot reuse them. if (query instanceof SecondaryIdQuery secondaryIdQuery) { final SecondaryIdDescriptionId selectedSecondaryId = secondaryIdQuery.getSecondaryId(); - final SecondaryIdDescriptionId reusedSecondaryId = ((SecondaryIdQuery) execution.getSubmitted()).getSecondaryId(); + final SecondaryIdDescriptionId reusedSecondaryId = ((SecondaryIdQuery) execution.getSubmitted()) + .getSecondaryId(); if (!selectedSecondaryId.equals(reusedSecondaryId)) { return null; @@ -458,9 +509,11 @@ private ManagedExecution tryReuse(QueryDescription query, ManagedExecutionId exe // If the user is not the owner of the execution, we definitely create a new Execution, so the owner can cancel it if (!user.isOwner(execution)) { - final ManagedExecution - newExecution = - executionManager.createExecution(execution.getSubmitted(), user.getId(), namespace, false); + final ManagedExecution newExecution = executionManager.createExecution( + execution.getSubmitted(), + user.getId(), + namespace, + false); newExecution.setLabel(execution.getLabel()); newExecution.setTags(execution.getTags().clone()); storage.updateExecution(newExecution); @@ -484,7 +537,10 @@ private ManagedExecution tryReuse(QueryDescription query, ManagedExecutionId exe /** * Execute a basic query on a single concept and return only the included entities Id's. */ - public Stream> resolveEntities(Subject subject, List> filters, DatasetId dataset) { + public Stream> resolveEntities( + Subject subject, + List> filters, + DatasetId dataset) { if (filters.stream().map(fv -> fv.getFilter().getConnector()).distinct().count() != 1) { throw new BadRequestException("Query exactly one connector at once."); } @@ -509,11 +565,16 @@ public Stream> resolveEntities(Subject subject, List> resolveEntities(Subject subject, List ids = config.getIdColumns() - .getIds().stream() - // We're only interested in returning printable ids - .filter(ColumnConfig::isPrint) - .collect(Collectors.toList()); + .getIds() + .stream() + // We're only interested in returning printable ids + .filter(ColumnConfig::isPrint) + .collect(Collectors.toList()); final Map id2index = IntStream.range(0, ids.size()) - .boxed() - .collect(Collectors.toMap( - idx -> ids.get(idx).getName(), - idx -> idx + .boxed() + .collect( + Collectors.toMap( + idx -> ids.get(idx).getName(), + idx -> idx )); final IdPrinter printer = IdColumnUtil.getIdPrinter(subject, execution, namespace, ids); // For each included entity emit a Map of { Id-Name -> Id-Value } - return result.streamResults(OptionalLong.empty()) - .map(printer::createId) - .map(entityPrintId -> { - final Map out = new HashMap<>(); - - for (Map.Entry entry : id2index.entrySet()) { - // Not all ExternalIds are expected to be set. - if (entityPrintId.getExternalId()[entry.getValue()] == null) { - continue; - } - - out.put(entry.getKey(), entityPrintId.getExternalId()[entry.getValue()]); - } + return result.streamResults(OptionalLong.empty()).map(printer::createId).map(entityPrintId -> { + final Map out = new HashMap<>(); + + for (Map.Entry entry : id2index.entrySet()) { + // Not all ExternalIds are expected to be set. + if (entityPrintId.getExternalId()[entry.getValue()] == null) { + continue; + } + + out.put(entry.getKey(), entityPrintId.getExternalId()[entry.getValue()]); + } - return out; - }) - .filter(Predicate.not(Map::isEmpty)); + return out; + }).filter(Predicate.not(Map::isEmpty)); } - public ResultStatistics getResultStatistics(ManagedExecutionId queryId, Subject subject) { + public ResultStatistics getResultStatistics( + ManagedExecutionId queryId, + Subject subject) { ManagedExecution maybeManagedQuery = queryId.resolve(); if (!(maybeManagedQuery instanceof SingleTableResult)) { - throw new BadRequestException("Statistics is only available for %s".formatted(SingleTableResult.class.getSimpleName())); + throw new BadRequestException( + "Statistics is only available for %s".formatted(SingleTableResult.class.getSimpleName())); } E managedQuery = (E) maybeManagedQuery; @@ -583,28 +646,36 @@ public ResultStatistics getResu final NumberFormat integerFormat = NumberFormat.getNumberInstance(locale); - final PrintSettings printSettings = - new PrintSettings(true, locale, config, null, null, decimalFormat, integerFormat); + final PrintSettings printSettings = new PrintSettings( + true, + locale, + config, + null, + null, + decimalFormat, + integerFormat); final UniqueNamer uniqueNamer = new UniqueNamer(printSettings); managedQuery.initExecutable(); final List resultInfos = managedQuery.collectResultInfos(); - final Optional - dateInfo = - resultInfos.stream().filter(info -> info.getSemantics().contains(new SemanticType.EventDateT())).findFirst(); + final Optional dateInfo = resultInfos.stream() + .filter( + info -> info.getSemantics().contains(new SemanticType.EventDateT())) + .findFirst(); final Optional dateIndex = dateInfo.map(resultInfos::indexOf); - return ResultStatistics.collectResultStatistics(managedQuery, - resultInfos, - dateInfo, - dateIndex, - printSettings, - uniqueNamer, - config, - new JavaResultPrinters() + return ResultStatistics.collectResultStatistics( + managedQuery, + resultInfos, + dateInfo, + dateIndex, + printSettings, + uniqueNamer, + config, + new JavaResultPrinters() ); } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/RequestAwareUriBuilder.java b/backend/src/main/java/com/bakdata/conquery/apiv1/RequestAwareUriBuilder.java index 825d2896142..996c5e79709 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/RequestAwareUriBuilder.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/RequestAwareUriBuilder.java @@ -4,6 +4,7 @@ import jakarta.servlet.http.HttpServletRequest; import jakarta.ws.rs.core.UriBuilder; + import lombok.experimental.UtilityClass; @UtilityClass diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/RequestHelper.java b/backend/src/main/java/com/bakdata/conquery/apiv1/RequestHelper.java index f004570c4a8..a119710dc67 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/RequestHelper.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/RequestHelper.java @@ -1,10 +1,10 @@ package com.bakdata.conquery.apiv1; -import java.net.URI; - import jakarta.servlet.http.HttpServletRequest; import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.MultivaluedMap; +import java.net.URI; + import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -23,12 +23,8 @@ public static String getRequestURL(HttpServletRequest req) { log.trace("Proto=`{}` Fwd-Host=`{}`", protocol, host); - return new URIBuilder() - .setHost(host) - .setScheme(protocol) - .toString(); - } - catch (Exception e) { + return new URIBuilder().setHost(host).setScheme(protocol).toString(); + } catch (Exception e) { log.warn("Failed to build response URL from X-Forward headers", e); } } @@ -51,12 +47,8 @@ public static URI getRequestURL(ContainerRequestContext req) { log.trace("Proto=`{}` Fwd-Host=`{}`", protocol, host); - return new URIBuilder() - .setHost(host) - .setScheme(protocol) - .build(); - } - catch (Exception e) { + return new URIBuilder().setHost(host).setScheme(protocol).build(); + } catch (Exception e) { log.warn("Failed to build response URL from X-Forward headers", e); } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/auth/PasswordHashCredential.java b/backend/src/main/java/com/bakdata/conquery/apiv1/auth/PasswordHashCredential.java index 38a44d5a004..bc247c4716a 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/auth/PasswordHashCredential.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/auth/PasswordHashCredential.java @@ -1,8 +1,9 @@ package com.bakdata.conquery.apiv1.auth; -import com.bakdata.conquery.io.cps.CPSType; import jakarta.validation.constraints.NotEmpty; +import com.bakdata.conquery.io.cps.CPSType; + @CPSType(base = CredentialType.class, id = "PASSWORD_HASH") public record PasswordHashCredential(@NotEmpty String hash) implements CredentialType { } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoRole.java b/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoRole.java index e0536c99b25..7e7bac6c2d7 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoRole.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoRole.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.apiv1.auth; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; import java.util.Collections; import java.util.Objects; import java.util.Set; @@ -9,8 +11,6 @@ import com.bakdata.conquery.models.auth.permissions.WildcardPermission; import com.bakdata.conquery.models.identifiable.ids.specific.RoleId; import com.fasterxml.jackson.annotation.JsonIgnore; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; import lombok.Builder; import lombok.Getter; import lombok.NonNull; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java b/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java index 525f712852a..f8352b95ba0 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java @@ -1,86 +1,86 @@ -package com.bakdata.conquery.apiv1.auth; - -import java.util.Collections; -import java.util.Set; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; - -import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.models.auth.UserManageable; -import com.bakdata.conquery.models.auth.basic.LocalAuthenticationRealm; -import com.bakdata.conquery.models.auth.entities.Role; -import com.bakdata.conquery.models.auth.entities.User; -import com.bakdata.conquery.models.auth.permissions.WildcardPermission; -import com.bakdata.conquery.models.identifiable.ids.specific.RoleId; -import com.bakdata.conquery.models.identifiable.ids.specific.UserId; -import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.Builder; -import lombok.Getter; -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; - -/** - * Factory class to create configured initial users. - */ -@Getter -@Builder -@Slf4j -public class ProtoUser { - - @NotEmpty - private final String name; - private final Set roles; - private String label; - /** - * String permissions in the form of - * {@link org.apache.shiro.authz.permission.WildcardPermission}, that the user - * should hold after initialization. - */ - @Builder.Default - @NotNull - private Set permissions = Collections.emptySet(); - - /** - * These are passed to realms that are able to manage users (implementing - * {@link UserManageable}, such as {@link LocalAuthenticationRealm}). - */ - @Builder.Default - @Valid - private CredentialType credential = null; - - public User createOrOverwriteUser(@NonNull MetaStorage storage) { - if (label == null) { - label = name; - } - - final User user = new User(name, label, storage); - storage.updateUser(user); - - if (roles != null){ - for (String roleId : roles) { - final Role role = storage.getRole(new RoleId(roleId)); - - if(role == null){ - log.warn("Unknown Role[{}] for {}", roleId, this); - continue; - } - - user.addRole(role.getId()); - } - } - - - for (String sPermission : permissions) { - user.addPermission(new WildcardPermission(sPermission)); - } - - - return user; - } - - @JsonIgnore - public UserId createId() { - return new UserId(name); - } -} +package com.bakdata.conquery.apiv1.auth; + +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; +import java.util.Collections; +import java.util.Set; + +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.models.auth.UserManageable; +import com.bakdata.conquery.models.auth.basic.LocalAuthenticationRealm; +import com.bakdata.conquery.models.auth.entities.Role; +import com.bakdata.conquery.models.auth.entities.User; +import com.bakdata.conquery.models.auth.permissions.WildcardPermission; +import com.bakdata.conquery.models.identifiable.ids.specific.RoleId; +import com.bakdata.conquery.models.identifiable.ids.specific.UserId; +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Builder; +import lombok.Getter; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; + +/** + * Factory class to create configured initial users. + */ +@Getter +@Builder +@Slf4j +public class ProtoUser { + + @NotEmpty + private final String name; + private final Set roles; + private String label; + /** + * String permissions in the form of + * {@link org.apache.shiro.authz.permission.WildcardPermission}, that the user + * should hold after initialization. + */ + @Builder.Default + @NotNull + private Set permissions = Collections.emptySet(); + + /** + * These are passed to realms that are able to manage users (implementing + * {@link UserManageable}, such as {@link LocalAuthenticationRealm}). + */ + @Builder.Default + @Valid + private CredentialType credential = null; + + public User createOrOverwriteUser(@NonNull MetaStorage storage) { + if (label == null) { + label = name; + } + + final User user = new User(name, label, storage); + storage.updateUser(user); + + if (roles != null) { + for (String roleId : roles) { + final Role role = storage.getRole(new RoleId(roleId)); + + if (role == null) { + log.warn("Unknown Role[{}] for {}", roleId, this); + continue; + } + + user.addRole(role.getId()); + } + } + + + for (String sPermission : permissions) { + user.addPermission(new WildcardPermission(sPermission)); + } + + + return user; + } + + @JsonIgnore + public UserId createId() { + return new UserId(name); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/auth/UsernamePasswordToken.java b/backend/src/main/java/com/bakdata/conquery/apiv1/auth/UsernamePasswordToken.java index cb8f3054d30..e594387da6d 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/auth/UsernamePasswordToken.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/auth/UsernamePasswordToken.java @@ -1,8 +1,9 @@ package com.bakdata.conquery.apiv1.auth; +import jakarta.validation.constraints.NotEmpty; + import com.bakdata.conquery.models.auth.basic.LocalAuthenticationRealm; import com.bakdata.conquery.models.identifiable.ids.specific.UserId; -import jakarta.validation.constraints.NotEmpty; import lombok.Data; /** diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java index b27c103f864..29ea5418fff 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java @@ -66,9 +66,7 @@ public class ExternalForm extends Form implements SubTyped { private final String subType; public JsonNode getExternalApiPayload() { - return ((ObjectNode) node.deepCopy() - .without("values")) - .set("type", new TextNode(subType)); + return ((ObjectNode) node.deepCopy().without("values")).set("type", new TextNode(subType)); } @@ -112,7 +110,9 @@ public String getLocalizedTypeLabel() { if (chosenLocale == null) { chosenLocale = localesFound.get(0); - log.trace("Locale lookup did not return a matching locale. Using the first title encountered: {}", chosenLocale); + log.trace( + "Locale lookup did not return a matching locale. Using the first title encountered: {}", + chosenLocale); } final JsonNode title = titleObj.path(chosenLocale.getLanguage()); @@ -134,7 +134,12 @@ public String getFormType() { } @Override - public ManagedExecution toManagedExecution(UserId user, DatasetId submittedDataset, MetaStorage storage, DatasetRegistry datasetRegistry, ConqueryConfig config) { + public ManagedExecution toManagedExecution( + UserId user, + DatasetId submittedDataset, + MetaStorage storage, + DatasetRegistry datasetRegistry, + ConqueryConfig config) { return new ExternalExecution(this, user, submittedDataset, storage, datasetRegistry, config); } @@ -179,7 +184,9 @@ public ExternalForm deserialize(JsonParser p, DeserializationContext ctxt) throw } @Override - public JsonDeserializer createContextual(DeserializationContext ctxt, BeanProperty property) throws JsonMappingException { + public JsonDeserializer createContextual( + DeserializationContext ctxt, + BeanProperty property) throws JsonMappingException { // This is only called once per typeId@SubTypeId final String subTypeId = (String) ctxt.getAttribute(CPSTypeIdResolver.ATTRIBUTE_SUB_TYPE); diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/Form.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/Form.java index 4a7d83bdd56..fb30c912336 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/Form.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/Form.java @@ -39,9 +39,12 @@ public String getFormType() { } - @Override - public void authorize(Subject subject, DatasetId submittedDataset, @NonNull List visitors, MetaStorage storage) { + public void authorize( + Subject subject, + DatasetId submittedDataset, + @NonNull List visitors, + MetaStorage storage) { QueryDescription.super.authorize(subject, submittedDataset, visitors, storage); // Check if subject is allowed to create this form final FormType formType = FormScanner.resolveFormType(getFormType()); @@ -54,7 +57,7 @@ public void authorize(Subject subject, DatasetId submittedDataset, @NonNull List } - /** + /** * Is called in context of a request to generate a default label. * If localization is needed use:
* diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/FormConfigAPI.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/FormConfigAPI.java index d82687fc102..7ddd74eda11 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/FormConfigAPI.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/FormConfigAPI.java @@ -1,9 +1,9 @@ package com.bakdata.conquery.apiv1.forms; -import java.time.LocalDateTime; -import java.util.UUID; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; +import java.time.LocalDateTime; +import java.util.UUID; import com.bakdata.conquery.models.forms.configs.FormConfig; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; @@ -27,14 +27,18 @@ public class FormConfigAPI { @NotEmpty private String formType; private String label; - @NotNull @Builder.Default + @NotNull + @Builder.Default private String[] tags = ArrayUtils.EMPTY_STRING_ARRAY; @NotNull private JsonNode values; - @VariableDefaultValue @NonNull @Builder.Default + @VariableDefaultValue + @NonNull + @Builder.Default private UUID formId = UUID.randomUUID(); - @VariableDefaultValue @Builder.Default + @VariableDefaultValue + @Builder.Default private LocalDateTime creationTime = LocalDateTime.now(); public FormConfig intern(UserId owner, DatasetId dataset) { diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/IndexPlacement.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/IndexPlacement.java index 1178cb0debe..d475c1bbd22 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/IndexPlacement.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/IndexPlacement.java @@ -3,9 +3,8 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; -@RequiredArgsConstructor @Getter +@RequiredArgsConstructor +@Getter public enum IndexPlacement { - BEFORE, - NEUTRAL, - AFTER; -} \ No newline at end of file + BEFORE, NEUTRAL, AFTER +} diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/AbsoluteMode.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/AbsoluteMode.java index b7528e56ef7..4bfb3f39980 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/AbsoluteMode.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/AbsoluteMode.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.apiv1.forms.export_form; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; import java.time.LocalDate; import java.util.List; import java.util.function.Consumer; @@ -15,8 +17,6 @@ import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.Visitable; import com.fasterxml.jackson.annotation.JsonView; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotNull; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; @@ -42,8 +42,9 @@ public class AbsoluteMode extends Mode { @Override public Query createSpecializedQuery() { - List resolutionsAndAlignments = - ExportForm.getResolutionAlignmentMap(getForm().getResolvedResolutions(), getAlignmentHint()); + List resolutionsAndAlignments = ExportForm.getResolutionAlignmentMap( + getForm().getResolvedResolutions(), + getAlignmentHint()); Query prerequisite = getForm().getPrerequisite(); return new AbsoluteFormQuery(prerequisite, dateRange, resolvedFeatures, resolutionsAndAlignments); diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/EntityDateMode.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/EntityDateMode.java index c382856d0b6..ab3107eed09 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/EntityDateMode.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/EntityDateMode.java @@ -1,68 +1,68 @@ -package com.bakdata.conquery.apiv1.forms.export_form; - -import java.time.LocalDate; -import java.util.function.Consumer; - -import javax.annotation.CheckForNull; - -import com.bakdata.conquery.apiv1.query.ArrayConceptQuery; -import com.bakdata.conquery.apiv1.query.Query; -import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.io.jackson.View; -import com.bakdata.conquery.models.common.Range; -import com.bakdata.conquery.models.common.daterange.CDateRange; -import com.bakdata.conquery.models.forms.managed.EntityDateQuery; -import com.bakdata.conquery.models.forms.util.Alignment; -import com.bakdata.conquery.models.query.DateAggregationMode; -import com.bakdata.conquery.models.query.QueryResolveContext; -import com.bakdata.conquery.models.query.Visitable; -import com.fasterxml.jackson.annotation.JsonView; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotNull; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.Setter; - -@Getter @Setter -@CPSType(id = "ENTITY_DATE", base = Mode.class) -public class EntityDateMode extends Mode { - - @CheckForNull - @Valid - private Range dateRange; - - @NotNull - private DateAggregationMode dateAggregationMode = DateAggregationMode.MERGE; - - - @Override - public void visit(Consumer visitor) { - } - - @NotNull - private Alignment alignmentHint = Alignment.QUARTER; - - - @JsonView(View.InternalCommunication.class) - @EqualsAndHashCode.Exclude - private ArrayConceptQuery resolvedFeatures; - - @Override - public void resolve(QueryResolveContext context) { - resolvedFeatures = ArrayConceptQuery.createFromFeatures(getForm().getFeatures()); - resolvedFeatures.resolve(context); - } - - @Override - public Query createSpecializedQuery() { - CDateRange dateRestriction = dateRange == null ? CDateRange.all() : CDateRange.of(dateRange); - - return new EntityDateQuery( - getForm().getPrerequisite(), - resolvedFeatures, - ExportForm.getResolutionAlignmentMap(getForm().getResolvedResolutions(), getAlignmentHint()), - dateRestriction, - dateAggregationMode - ); - } -} +package com.bakdata.conquery.apiv1.forms.export_form; + +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; +import java.time.LocalDate; +import java.util.function.Consumer; +import javax.annotation.CheckForNull; + +import com.bakdata.conquery.apiv1.query.ArrayConceptQuery; +import com.bakdata.conquery.apiv1.query.Query; +import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.io.jackson.View; +import com.bakdata.conquery.models.common.Range; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.forms.managed.EntityDateQuery; +import com.bakdata.conquery.models.forms.util.Alignment; +import com.bakdata.conquery.models.query.DateAggregationMode; +import com.bakdata.conquery.models.query.QueryResolveContext; +import com.bakdata.conquery.models.query.Visitable; +import com.fasterxml.jackson.annotation.JsonView; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +@CPSType(id = "ENTITY_DATE", base = Mode.class) +public class EntityDateMode extends Mode { + + @CheckForNull + @Valid + private Range dateRange; + + @NotNull + private DateAggregationMode dateAggregationMode = DateAggregationMode.MERGE; + + + @Override + public void visit(Consumer visitor) { + } + + @NotNull + private Alignment alignmentHint = Alignment.QUARTER; + + + @JsonView(View.InternalCommunication.class) + @EqualsAndHashCode.Exclude + private ArrayConceptQuery resolvedFeatures; + + @Override + public void resolve(QueryResolveContext context) { + resolvedFeatures = ArrayConceptQuery.createFromFeatures(getForm().getFeatures()); + resolvedFeatures.resolve(context); + } + + @Override + public Query createSpecializedQuery() { + CDateRange dateRestriction = dateRange == null ? CDateRange.all() : CDateRange.of(dateRange); + + return new EntityDateQuery( + getForm().getPrerequisite(), + resolvedFeatures, + ExportForm.getResolutionAlignmentMap(getForm().getResolvedResolutions(), getAlignmentHint()), + dateRestriction, + dateAggregationMode + ); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/ExportForm.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/ExportForm.java index 23d96c80217..e5a101dba6d 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/ExportForm.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/ExportForm.java @@ -1,5 +1,9 @@ package com.bakdata.conquery.apiv1.forms.export_form; +import jakarta.validation.Valid; +import jakarta.validation.ValidationException; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; import java.util.Collections; import java.util.List; import java.util.Map; @@ -7,10 +11,6 @@ import java.util.function.Consumer; import java.util.stream.Collectors; import javax.annotation.Nullable; -import jakarta.validation.Valid; -import jakarta.validation.ValidationException; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; import c10n.C10N; import com.bakdata.conquery.ConqueryConstants; @@ -104,15 +104,18 @@ public class ExportForm extends Form implements InternalForm { * Note that this alignment is chosen when a resolution is equal or coarser. * @return The given resolutions mapped to a fitting calendar alignment. */ - public static List getResolutionAlignmentMap(List resolutions, Alignment alignmentHint) { + public static List getResolutionAlignmentMap( + List resolutions, + Alignment alignmentHint) { return resolutions.stream() - .map(r -> ResolutionAndAlignment.of(r, getFittingAlignment(alignmentHint, r))) - .collect(Collectors.toList()); + .map( + r -> ResolutionAndAlignment.of(r, getFittingAlignment(alignmentHint, r))) + .collect(Collectors.toList()); } private static Alignment getFittingAlignment(Alignment alignmentHint, Resolution resolution) { - if(resolution.isAlignmentSupported(alignmentHint) ) { + if (resolution.isAlignmentSupported(alignmentHint)) { return alignmentHint; } return resolution.getDefaultAlignment(); @@ -128,8 +131,8 @@ public void visit(Consumer visitor) { @Override public Map createSubQueries() { return Map.of( - ConqueryConstants.SINGLE_RESULT_TABLE_NAME, - timeMode.createSpecializedQuery() + ConqueryConstants.SINGLE_RESULT_TABLE_NAME, + timeMode.createSpecializedQuery() ); } @@ -139,8 +142,12 @@ public String getLocalizedTypeLabel() { } @Override - public ManagedInternalForm toManagedExecution(UserId user, DatasetId submittedDataset, MetaStorage storage, DatasetRegistry datasetRegistry, - ConqueryConfig config) { + public ManagedInternalForm toManagedExecution( + UserId user, + DatasetId submittedDataset, + MetaStorage storage, + DatasetRegistry datasetRegistry, + ConqueryConfig config) { return new ManagedInternalForm<>(this, user, submittedDataset, storage, datasetRegistry, config); } @@ -155,11 +162,10 @@ public Set collectRequiredQueries() { @Override public void resolve(QueryResolveContext context) { - if(queryGroupId != null) { + if (queryGroupId != null) { queryGroup = (ManagedQuery) queryGroupId.resolve(); prerequisite = queryGroup.getQuery(); - } - else { + } else { prerequisite = new ConceptQuery(new CQYes()); } @@ -170,18 +176,19 @@ public void resolve(QueryResolveContext context) { timeMode.resolve(context); List resolutionsFlat = resolution.stream() - .flatMap(ResolutionShortNames::correspondingResolutions) - .distinct() - .toList(); + .flatMap( + ResolutionShortNames::correspondingResolutions) + .distinct() + .toList(); if (isAlsoCreateCoarserSubdivisions()) { if (resolutionsFlat.size() != 1) { - throw new IllegalStateException("Abort Form creation, because coarser subdivision are requested and multiple resolutions are given. With 'alsoCreateCoarserSubdivisions' set to true, provide only one resolution."); + throw new IllegalStateException( + "Abort Form creation, because coarser subdivision are requested and multiple resolutions are given. With 'alsoCreateCoarserSubdivisions' set to true, provide only one resolution."); } resolvedResolutions = resolutionsFlat.getFirst().getThisAndCoarserSubdivisions(); - } - else { + } else { resolvedResolutions = resolutionsFlat; } } @@ -193,11 +200,12 @@ public interface DefaultSelectSettable { static void enable(CQElement feature) { switch (feature) { - case DefaultSelectSettable settable -> settable.setDefaultSelects(); + case DefaultSelectSettable settable -> settable.setDefaultSelects(); // CQNegation and CQDateRestriction chain CQElements and don't have selects themselves case CQNegation negation -> enable(negation.getChild()); case CQDateRestriction dr -> enable(dr.getChild()); - default -> {} + default -> { + } } } @@ -220,9 +228,10 @@ public static class ResolutionAndAlignment { private final Alignment alignment; @JsonCreator - public static ResolutionAndAlignment of(Resolution resolution, Alignment alignment){ + public static ResolutionAndAlignment of(Resolution resolution, Alignment alignment) { if (!resolution.isAlignmentSupported(alignment)) { - throw new ValidationException(String.format("The alignment %s is not supported by the resolution %s", alignment, resolution)); + throw new ValidationException( + String.format("The alignment %s is not supported by the resolution %s", alignment, resolution)); } return new ResolutionAndAlignment(resolution, alignment); diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/FullExportForm.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/FullExportForm.java index de4c81ae1a3..5438f69cbae 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/FullExportForm.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/FullExportForm.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.apiv1.forms.export_form; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; import java.time.LocalDate; import java.util.Collections; import java.util.List; @@ -7,8 +9,6 @@ import java.util.Set; import java.util.function.Consumer; import javax.annotation.Nullable; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotEmpty; import c10n.C10N; import com.bakdata.conquery.ConqueryConstants; @@ -88,8 +88,7 @@ public Map createSubQueries() { if (queryGroupId != null) { query = queryGroup.getQuery(); - } - else { + } else { query = new ConceptQuery(new CQYes()); } @@ -100,8 +99,8 @@ public Map createSubQueries() { return Map.of( - ConqueryConstants.SINGLE_RESULT_TABLE_NAME, - exportQuery + ConqueryConstants.SINGLE_RESULT_TABLE_NAME, + exportQuery ); } @@ -129,8 +128,12 @@ public String getLocalizedTypeLabel() { @Override - public ManagedInternalForm toManagedExecution(UserId user, DatasetId submittedDataset, MetaStorage storage, DatasetRegistry datasetRegistry, - ConqueryConfig config) { + public ManagedInternalForm toManagedExecution( + UserId user, + DatasetId submittedDataset, + MetaStorage storage, + DatasetRegistry datasetRegistry, + ConqueryConfig config) { return new ManagedInternalForm<>(this, user, submittedDataset, storage, datasetRegistry, config); } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/RelativeMode.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/RelativeMode.java index 852a88f6868..aafdc62adec 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/RelativeMode.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/RelativeMode.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.apiv1.forms.export_form; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotNull; import java.util.function.Consumer; import com.bakdata.conquery.apiv1.forms.IndexPlacement; @@ -14,13 +16,12 @@ import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.Visitable; import com.fasterxml.jackson.annotation.JsonView; -import jakarta.validation.constraints.Min; -import jakarta.validation.constraints.NotNull; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; -@Getter @Setter +@Getter +@Setter @CPSType(id = "RELATIVE", base = Mode.class) @EqualsAndHashCode(callSuper = true) public class RelativeMode extends Mode { diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendConfiguration.java b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendConfiguration.java index 1a75ccf7693..a8be0564db9 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendConfiguration.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendConfiguration.java @@ -17,11 +17,11 @@ * @param contactEmail typical a mailto-url */ public record FrontendConfiguration( - List versions, - FrontendConfig.CurrencyConfig currency, - IdColumnConfig queryUpload, - URL manualUrl, - String contactEmail, - LocalDate observationPeriodStart + List versions, + FrontendConfig.CurrencyConfig currency, + IdColumnConfig queryUpload, + URL manualUrl, + String contactEmail, + LocalDate observationPeriodStart ) { } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendFilterConfiguration.java b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendFilterConfiguration.java index 28b8b70d4a3..219eb38d561 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendFilterConfiguration.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendFilterConfiguration.java @@ -1,13 +1,12 @@ package com.bakdata.conquery.apiv1.frontend; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; import java.util.Collections; import java.util.List; import java.util.Map; - import javax.annotation.Nullable; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.apiv1.FilterTemplate; import com.bakdata.conquery.models.identifiable.ids.specific.FilterId; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendList.java b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendList.java index fc3e657afcf..3bf92bfbe38 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendList.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendList.java @@ -1,9 +1,9 @@ package com.bakdata.conquery.apiv1.frontend; +import jakarta.ws.rs.core.EntityTag; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import jakarta.ws.rs.core.EntityTag; import com.bakdata.conquery.models.identifiable.ids.specific.ConceptElementId; import com.fasterxml.jackson.core.JsonGenerator; @@ -37,7 +37,10 @@ public void serialize(JsonGenerator gen, SerializerProvider serializers) throws @Override - public void serializeWithType(JsonGenerator gen, SerializerProvider serializers, TypeSerializer typeSer) throws IOException { + public void serializeWithType( + JsonGenerator gen, + SerializerProvider serializers, + TypeSerializer typeSer) throws IOException { serialize(gen, serializers); } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendNode.java b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendNode.java index 1f568140d14..798037dcb97 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendNode.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendNode.java @@ -1,9 +1,9 @@ package com.bakdata.conquery.apiv1.frontend; -import java.time.LocalDate; -import java.util.List; import jakarta.validation.Valid; import jakarta.validation.constraints.NotNull; +import java.time.LocalDate; +import java.util.List; import com.bakdata.conquery.apiv1.KeyValue; import com.bakdata.conquery.models.common.Range; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendResultType.java b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendResultType.java index 22d81d59370..cb36956d7f8 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendResultType.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendResultType.java @@ -21,9 +21,7 @@ public static class List extends FrontendResultType { public static FrontendResultType from(ResultType resultType) { if (resultType instanceof ResultType.ListT listT) { - return List.builder() - .elementType(from(listT.getElementType())) - .type("LIST").build(); + return List.builder().elementType(from(listT.getElementType())).type("LIST").build(); } return FrontendResultType.builder().type(resultType.typeInfo()).build(); diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendRoot.java b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendRoot.java index a7e6da657fd..627abc68194 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendRoot.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendRoot.java @@ -1,10 +1,10 @@ package com.bakdata.conquery.apiv1.frontend; +import jakarta.validation.Valid; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; -import jakarta.validation.Valid; import com.bakdata.conquery.models.identifiable.ids.Id; import lombok.Getter; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendTable.java b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendTable.java index 3db175c7c26..f8f2aee09d5 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendTable.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendTable.java @@ -1,11 +1,10 @@ package com.bakdata.conquery.apiv1.frontend; -import java.util.List; -import java.util.Set; - import jakarta.validation.Valid; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; +import java.util.List; +import java.util.Set; import com.bakdata.conquery.models.identifiable.ids.specific.ConnectorId; import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendValidityDate.java b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendValidityDate.java index 99772d402b3..26199f6bf46 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendValidityDate.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendValidityDate.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.apiv1.frontend; import java.util.List; - import javax.annotation.Nullable; import lombok.AllArgsConstructor; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendValue.java b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendValue.java index 4310ff5b2c5..a779b456576 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendValue.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendValue.java @@ -15,8 +15,8 @@ @Data @EqualsAndHashCode(onlyExplicitlyIncluded = true) public class FrontendValue implements Comparable { - private static final Comparator COMPARATOR = Comparator.comparing(FrontendValue::getValue) - .thenComparing(FrontendValue::getLabel); + private static final Comparator COMPARATOR = Comparator.comparing( + FrontendValue::getValue).thenComparing(FrontendValue::getLabel); /** * Value is the only relevant data-point for hashing/equality and searching from the service perspective. diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/VersionContainer.java b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/VersionContainer.java index e2739ec9ce1..8152bdee8ea 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/VersionContainer.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/VersionContainer.java @@ -5,8 +5,8 @@ import lombok.NonNull; public record VersionContainer( - @NonNull String name, - String version, - ZonedDateTime buildTime + @NonNull String name, + String version, + ZonedDateTime buildTime ) { } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/ArrayConceptQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/ArrayConceptQuery.java index ed1cac3541e..2c4974b6cbd 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/ArrayConceptQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/ArrayConceptQuery.java @@ -1,13 +1,13 @@ package com.bakdata.conquery.apiv1.query; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.function.Consumer; import java.util.stream.Collectors; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.jackson.View; @@ -54,9 +54,7 @@ public class ArrayConceptQuery extends Query { protected DateAggregationMode resolvedDateAggregationMode; public static ArrayConceptQuery createFromFeatures(List features) { - List cqWraps = features.stream() - .map(ConceptQuery::new) - .collect(Collectors.toList()); + List cqWraps = features.stream().map(ConceptQuery::new).collect(Collectors.toList()); return new ArrayConceptQuery(cqWraps); } @@ -73,7 +71,10 @@ public ArrayConceptQuery(List queries) { public void resolve(QueryResolveContext context) { resolvedDateAggregationMode = dateAggregationMode; if (context.getDateAggregationMode() != null) { - log.trace("Overriding date aggregation mode ({}) with mode from context ({})", dateAggregationMode, context.getDateAggregationMode()); + log.trace( + "Overriding date aggregation mode ({}) with mode from context ({})", + dateAggregationMode, + context.getDateAggregationMode()); resolvedDateAggregationMode = context.getDateAggregationMode(); } childQueries.forEach(c -> c.resolve(context.withDateAggregationMode(resolvedDateAggregationMode))); @@ -97,7 +98,7 @@ public List getResultInfos() { final List resultInfos = new ArrayList<>(); ResultInfo dateInfo = ResultHeaders.datesInfo(); - if(getResolvedDateAggregationMode() != DateAggregationMode.NONE){ + if (getResolvedDateAggregationMode() != DateAggregationMode.NONE) { // Add one DateInfo for the whole Query resultInfos.addFirst(dateInfo); } @@ -105,10 +106,11 @@ public List getResultInfos() { childQueries.forEach(q -> resultInfos.addAll(q.getResultInfos())); - if(!resultInfos.isEmpty()) { - // Remove DateInfo from each childQuery + if (!resultInfos.isEmpty()) { + // Remove DateInfo from each childQuery resultInfos.subList(lastIndex, resultInfos.size()) - .removeIf(resultInfo -> resultInfo.getSemantics().contains(new SemanticType.EventDateT())); + .removeIf( + resultInfo -> resultInfo.getSemantics().contains(new SemanticType.EventDateT())); } return resultInfos; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQElement.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQElement.java index 50ab2c4ff34..abfb4078cd4 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQElement.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQElement.java @@ -1,11 +1,11 @@ package com.bakdata.conquery.apiv1.query; +import jakarta.validation.constraints.NotNull; import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Set; import java.util.function.Consumer; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.commands.ShardNode; @@ -78,7 +78,7 @@ public final Set collectRequiredQueries() { return set; } - public abstract void collectRequiredQueries(Set requiredQueries) ; + public abstract void collectRequiredQueries(Set requiredQueries); @JsonIgnore public abstract List getResultInfos(); diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/ConceptQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/ConceptQuery.java index c01dbecf3d1..1fb8e944b6c 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/ConceptQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/ConceptQuery.java @@ -1,11 +1,11 @@ package com.bakdata.conquery.apiv1.query; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.function.Consumer; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.jackson.View; @@ -71,7 +71,10 @@ public void resolve(QueryResolveContext context) { resolvedDateAggregationMode = dateAggregationMode; if (context.getDateAggregationMode() != null) { - log.trace("Overriding date aggregation mode ({}) with mode from context ({})", dateAggregationMode, context.getDateAggregationMode()); + log.trace( + "Overriding date aggregation mode ({}) with mode from context ({})", + dateAggregationMode, + context.getDateAggregationMode()); resolvedDateAggregationMode = context.getDateAggregationMode(); } root.resolve(context.withDateAggregationMode(resolvedDateAggregationMode)); @@ -107,4 +110,4 @@ public CQElement getReusableComponents() { public RequiredEntities collectRequiredEntities(QueryExecutionContext context) { return getRoot().collectRequiredEntities(context); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/ExternalUpload.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/ExternalUpload.java index f4a25b348c2..794ef101d93 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/ExternalUpload.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/ExternalUpload.java @@ -3,19 +3,18 @@ import java.util.Arrays; import java.util.List; -import com.bakdata.conquery.models.auth.entities.User; +import com.bakdata.conquery.models.auth.entities.Subject; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; -import lombok.RequiredArgsConstructor; import lombok.Setter; /** - * Data Container for API call of {@link com.bakdata.conquery.resources.api.QueryResource#upload(User, ExternalUpload)}. + * Data Container for API call of {@link com.bakdata.conquery.resources.api.DatasetQueryResource#upload(Subject, ExternalUpload)} * * This class acts as a wrapper for {@link com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal}. */ diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/Query.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/Query.java index 2295e12d146..6d2eef51f52 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/Query.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/Query.java @@ -40,7 +40,12 @@ public Set collectRequiredQueries() { public abstract List getResultInfos(); @Override - public ManagedQuery toManagedExecution(UserId user, DatasetId submittedDataset, MetaStorage storage, DatasetRegistry datasetRegistry, ConqueryConfig config) { + public ManagedQuery toManagedExecution( + UserId user, + DatasetId submittedDataset, + MetaStorage storage, + DatasetRegistry datasetRegistry, + ConqueryConfig config) { return new ManagedQuery(this, user, submittedDataset, storage, datasetRegistry, config); } @@ -59,8 +64,6 @@ public CQElement getReusableComponents() { * @return the number of results in the result List. */ public long countResults(Stream results) { - return results.map(EntityResult::listResultLines) - .mapToLong(List::size) - .sum(); + return results.map(EntityResult::listResultLines).mapToLong(List::size).sum(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/QueryDescription.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/QueryDescription.java index a045a80d690..8c0ad78fab3 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/QueryDescription.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/QueryDescription.java @@ -47,7 +47,12 @@ public interface QueryDescription extends Visitable { * @param config * @return */ - ManagedExecution toManagedExecution(UserId user, DatasetId submittedDataset, MetaStorage storage, DatasetRegistry datasetRegistry, ConqueryConfig config); + ManagedExecution toManagedExecution( + UserId user, + DatasetId submittedDataset, + MetaStorage storage, + DatasetRegistry datasetRegistry, + ConqueryConfig config); /** * Initializes a submitted description using the provided context. @@ -55,7 +60,7 @@ public interface QueryDescription extends Visitable { * @param context Holds information which can be used for the initialize the description of the query to be executed. */ void resolve(QueryResolveContext context); - + /** * Allows the implementation to add visitors that traverse the QueryTree. * All visitors are concatenated so only a single traverse needs to be done. @@ -65,39 +70,57 @@ default void addVisitors(@NonNull List visitors) { // Register visitors for permission checks visitors.add(new QueryUtils.ExternalIdChecker()); } - + /** * Check implementation specific permissions. Is called after all visitors have been registered and executed. */ - default void authorize(Subject subject, DatasetId submittedDataset, List visitors, MetaStorage storage) { + default void authorize( + Subject subject, + DatasetId submittedDataset, + List visitors, + MetaStorage storage) { authorizeQuery(this, subject, submittedDataset, visitors, storage); } - static void authorizeQuery(QueryDescription queryDescription, Subject subject, DatasetId submittedDataset, List visitors, MetaStorage storage) { - NamespacedIdentifiableCollector nsIdCollector = QueryUtils.getVisitor(visitors, NamespacedIdentifiableCollector.class); + static void authorizeQuery( + QueryDescription queryDescription, + Subject subject, + DatasetId submittedDataset, + List visitors, + MetaStorage storage) { + NamespacedIdentifiableCollector nsIdCollector = QueryUtils.getVisitor( + visitors, + NamespacedIdentifiableCollector.class); ExternalIdChecker externalIdChecker = QueryUtils.getVisitor(visitors, ExternalIdChecker.class); // Generate DatasetPermissions - final Set datasets = nsIdCollector.getIdentifiables().stream() - .map(NamespacedIdentifiable::getDataset) - .collect(Collectors.toSet()); + final Set datasets = nsIdCollector.getIdentifiables() + .stream() + .map( + NamespacedIdentifiable::getDataset) + .collect(Collectors.toSet()); subject.authorize(datasets, Ability.READ); // Generate ConceptPermissions - final Set> concepts = nsIdCollector.getIdentifiables().stream() - .filter(ConceptElement.class::isInstance) - .map(ConceptElement.class::cast) - .>map(ConceptElement::getConcept) - .collect(Collectors.toSet()); + final Set> concepts = nsIdCollector.getIdentifiables() + .stream() + .filter( + ConceptElement.class::isInstance) + .map(ConceptElement.class::cast) + .>map( + ConceptElement::getConcept) + .collect(Collectors.toSet()); subject.authorize(concepts, Ability.READ); // Check reused query permissions - final Set collectedExecutions = queryDescription.collectRequiredQueries().stream() - .map(storage::getExecution) - .filter(Objects::nonNull) - .collect(Collectors.toSet()); + final Set collectedExecutions = queryDescription.collectRequiredQueries() + .stream() + .map( + storage::getExecution) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); subject.authorize(collectedExecutions, Ability.READ); @@ -109,7 +132,7 @@ static void authorizeQuery(QueryDescription queryDescription, Subject subject, D Set collectRequiredQueries(); - default RequiredEntities collectRequiredEntities(QueryExecutionContext context){ + default RequiredEntities collectRequiredEntities(QueryExecutionContext context) { return new RequiredEntities(context.getBucketManager().getEntities()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/ResultHeaders.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/ResultHeaders.java index 3394c7e101e..254c4e6a73a 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/ResultHeaders.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/ResultHeaders.java @@ -23,7 +23,7 @@ public static ResultInfo datesInfo() { final ResultType.ListT type = new ResultType.ListT<>(ResultType.Primitive.DATE_RANGE); - return new FixedLabelResultInfo(type, Set.of(new SemanticType.EventDateT())){ + return new FixedLabelResultInfo(type, Set.of(new SemanticType.EventDateT())) { @Override public String userColumnName(PrintSettings printSettings) { return C10nCache.getLocalized(ResultHeadersC10n.class, printSettings.getLocale()).dates(); @@ -54,7 +54,9 @@ public ResultSetProcessor.Reader createReader(ResultSetProcessor resultSetPro } public static ResultInfo sourceInfo() { - return new FixedLabelResultInfo(ResultType.Primitive.STRING, Set.of(new SemanticType.SourcesT(), new SemanticType.CategoricalT(), new SemanticType.GroupT())) { + return new FixedLabelResultInfo( + ResultType.Primitive.STRING, + Set.of(new SemanticType.SourcesT(), new SemanticType.CategoricalT(), new SemanticType.GroupT())) { @Override public String userColumnName(PrintSettings printSettings) { return C10nCache.getLocalized(ResultHeadersC10n.class, printSettings.getLocale()).source(); diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java index ed18c4234cd..3f12440d1c6 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java @@ -1,12 +1,12 @@ package com.bakdata.conquery.apiv1.query; +import jakarta.validation.constraints.NotNull; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.function.Consumer; import java.util.stream.Collectors; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; @@ -63,13 +63,17 @@ public class SecondaryIdQuery extends Query { @Override public SecondaryIdQueryPlan createQueryPlan(QueryPlanContext context) { final SecondaryIdDescription secondaryIdDescription = secondaryId.resolve(); - final ConceptQueryPlan queryPlan = query.createQueryPlan(context.withSelectedSecondaryId(secondaryIdDescription)); - - return new SecondaryIdQueryPlan(query, context, secondaryIdDescription, withSecondaryId.stream() - .map(ColumnId::resolve) - .collect(Collectors.toSet()), withoutSecondaryId.stream() - .map(TableId::resolve) - .collect(Collectors.toSet()), queryPlan, context.getSecondaryIdSubPlanRetention()); + final ConceptQueryPlan queryPlan = query.createQueryPlan( + context.withSelectedSecondaryId(secondaryIdDescription)); + + return new SecondaryIdQueryPlan( + query, + context, + secondaryIdDescription, + withSecondaryId.stream().map(ColumnId::resolve).collect(Collectors.toSet()), + withoutSecondaryId.stream().map(TableId::resolve).collect(Collectors.toSet()), + queryPlan, + context.getSecondaryIdSubPlanRetention()); } @Override @@ -83,7 +87,10 @@ public void resolve(final QueryResolveContext context) { DateAggregationMode resolvedDateAggregationMode = dateAggregationMode; if (context.getDateAggregationMode() != null) { - log.trace("Overriding date aggregation mode ({}) with mode from context ({})", dateAggregationMode, context.getDateAggregationMode()); + log.trace( + "Overriding date aggregation mode ({}) with mode from context ({})", + dateAggregationMode, + context.getDateAggregationMode()); resolvedDateAggregationMode = context.getDateAggregationMode(); } final QueryResolveContext resolvedContext = context.withDateAggregationMode(resolvedDateAggregationMode); @@ -112,8 +119,7 @@ public void resolve(final QueryResolveContext context) { if (secondaryIdColumn != null && !concept.isExcludeFromSecondaryId()) { withSecondaryId.add(secondaryIdColumn.getId()); - } - else { + } else { withoutSecondaryId.add(table.getId()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java index d0dfefa6e20..ba2fb9c48ab 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java @@ -1,5 +1,8 @@ package com.bakdata.conquery.apiv1.query; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; import java.time.LocalDate; import java.util.ArrayList; import java.util.Arrays; @@ -13,9 +16,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.stream.Collectors; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; @@ -131,7 +131,12 @@ public TableExportQueryPlan createQueryPlan(QueryPlanContext context) { } } - return new TableExportQueryPlan(query.createQueryPlan(context), CDateSet.create(CDateRange.of(dateRange)), filterQueryNodes, positions, rawConceptValues); + return new TableExportQueryPlan( + query.createQueryPlan(context), + CDateSet.create(CDateRange.of(dateRange)), + filterQueryNodes, + positions, + rawConceptValues); } @Override @@ -149,56 +154,68 @@ public void resolve(QueryResolveContext context) { // We need to know if a column is a concept column, so we can prioritize it, if it is also a SecondaryId conceptColumns = concepts.stream() - .map(CQConcept::getTables) - .flatMap(Collection::stream) - .map(CQTable::getConnector) - .map(ConnectorId::resolve) - .map(Connector::getColumn) - .filter(Objects::nonNull) - .collect(Collectors.toSet()); + .map(CQConcept::getTables) + .flatMap(Collection::stream) + .map( + CQTable::getConnector) + .map(ConnectorId::resolve) + .map(Connector::getColumn) + .filter(Objects::nonNull) + .collect( + Collectors.toSet()); secondaryIdPositions = calculateSecondaryIdPositions(currentPosition, conceptColumns, concepts); final Set validityDates = concepts.stream() - .map(CQConcept::getTables) - .flatMap(Collection::stream) - .map(CQTable::findValidityDate) - .filter(Objects::nonNull) - .collect(Collectors.toSet()); - - - positions = calculateColumnPositions(currentPosition, concepts, secondaryIdPositions, conceptColumns, validityDates); + .map(CQConcept::getTables) + .flatMap( + Collection::stream) + .map(CQTable::findValidityDate) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); + + + positions = calculateColumnPositions( + currentPosition, + concepts, + secondaryIdPositions, + conceptColumns, + validityDates); } private static Map calculateSecondaryIdPositions( - AtomicInteger currentPosition, - Set conceptColumns, - List tables) { + AtomicInteger currentPosition, + Set conceptColumns, + List tables) { final Map secondaryIdPositions = new HashMap<>(); // SecondaryIds are pulled to the front and grouped over all tables tables.stream() - .flatMap(con -> con.getTables().stream()) - .flatMap(table -> Arrays.stream(table.getConnector().resolve().getResolvedTable().getColumns())) - // Concept Columns are placed separately so they won't provide a secondaryId - .filter(col -> !conceptColumns.contains(col.getId())) - .map(Column::getSecondaryId) - .filter(Objects::nonNull) - .map(SecondaryIdDescriptionId::resolve) - .distinct() - .sorted(Comparator.comparing(SecondaryIdDescription::getLabel)) - // Using for each and not a collector allows us to guarantee sorted insertion. - .forEach(secondaryId -> secondaryIdPositions.put(secondaryId.getId(), currentPosition.getAndIncrement())); + .flatMap(con -> con.getTables().stream()) + .flatMap( + table -> Arrays.stream(table.getConnector().resolve().getResolvedTable().getColumns())) + // Concept Columns are placed separately so they won't provide a secondaryId + .filter(col -> !conceptColumns.contains(col.getId())) + .map(Column::getSecondaryId) + .filter( + Objects::nonNull) + .map(SecondaryIdDescriptionId::resolve) + .distinct() + .sorted( + Comparator.comparing(SecondaryIdDescription::getLabel)) + // Using for each and not a collector allows us to guarantee sorted insertion. + .forEach( + secondaryId -> secondaryIdPositions.put(secondaryId.getId(), currentPosition.getAndIncrement())); return secondaryIdPositions; } private static Map calculateColumnPositions( - AtomicInteger currentPosition, - List tables, - Map secondaryIdPositions, - Collection conceptColumns, - Collection validityDates) { + AtomicInteger currentPosition, + List tables, + Map secondaryIdPositions, + Collection conceptColumns, + Collection validityDates) { final Map positions = new HashMap<>(); @@ -256,13 +273,17 @@ private List createResultInfos(Set conceptColumns) { final Map connectorColumns = concepts.stream() - .flatMap(con -> con.getTables().stream()) - .map(CQTable::getConnector) - .map(ConnectorId::resolve) - .filter(con -> con.getColumn() != null) - .collect(Collectors.toMap(Connector::getColumn, - (Connector connector) -> connector.getConcept().getId() - )); + .flatMap( + con -> con.getTables().stream()) + .map(CQTable::getConnector) + .map(ConnectorId::resolve) + .filter( + con -> con.getColumn() != null) + .collect( + Collectors.toMap( + Connector::getColumn, + (Connector connector) -> connector.getConcept().getId() + )); for (Map.Entry entry : positions.entrySet()) { @@ -278,7 +299,8 @@ private List createResultInfos(Set conceptColumns) { // SecondaryIds and date columns are pulled to the front, thus already covered. if (column.getSecondaryId() != null && !conceptColumns.contains(columnId)) { - infos[secondaryIdPositions.get(column.getSecondaryId())].addSemantics(new SemanticType.ColumnT(columnId)); + infos[secondaryIdPositions.get(column.getSecondaryId())].addSemantics( + new SemanticType.ColumnT(columnId)); continue; } @@ -287,17 +309,23 @@ private List createResultInfos(Set conceptColumns) { final ConceptId concept = connectorColumns.get(columnId); // Additionally, Concept Columns are returned as ConceptElementId, when rawConceptColumns is not set. - columnResultInfo = - new ColumnResultInfo(column, ResultType.Primitive.STRING, column.getDescription(), isRawConceptValues() ? null : (TreeConcept) concept.resolve()); + columnResultInfo = new ColumnResultInfo( + column, + ResultType.Primitive.STRING, + column.getDescription(), + isRawConceptValues() ? null : (TreeConcept) concept.resolve()); // Columns that are used to build concepts are marked as ConceptColumn. columnResultInfo.addSemantics(new SemanticType.ConceptColumnT(concept)); infos[position] = columnResultInfo; - } - else { + } else { // If it's not a connector column, we just link to the source column. - columnResultInfo = new ColumnResultInfo(column, ResultType.resolveResultType(column.getType()), column.getDescription(), null); + columnResultInfo = new ColumnResultInfo( + column, + ResultType.resolveResultType(column.getType()), + column.getDescription(), + null); columnResultInfo.addSemantics(new SemanticType.ColumnT(column.getId())); } @@ -333,12 +361,11 @@ public boolean isConceptColumnsSingletons() { continue; } connectorColumns.computeIfAbsent(resolve.getColumn(), ignored -> new ArrayList<>()) - .add(cqTable.getConnector().getConcept()); + .add( + cqTable.getConnector().getConcept()); } } - return connectorColumns.values() - .stream() - .map(List::size).allMatch(s -> s == 1); + return connectorColumns.values().stream().map(List::size).allMatch(s -> s == 1); } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TemporalSamplerFactory.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TemporalSamplerFactory.java index 6cc54b7d2e4..a1503318eb2 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TemporalSamplerFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TemporalSamplerFactory.java @@ -49,7 +49,7 @@ public Sampler sampler(int today) { return OptionalInt.of(span.getMaxValue()); } - if(span.hasLowerBound() && span.getMinValue() > today){ + if (span.hasLowerBound() && span.getMinValue() > today) { return OptionalInt.of(span.getMinValue()); } @@ -82,8 +82,7 @@ public Sampler sampler(int today) { if (span.hasLowerBound()) { lower = span.getMinValue(); - } - else { + } else { // Very rare edge case with no real answer to give lower = CDateRange.NEGATIVE_INFINITY; } @@ -93,8 +92,7 @@ public Sampler sampler(int today) { if (span.hasUpperBound()) { upper = span.getMaxValue(); - } - else { + } else { // Since there is no satisfying real value, but real values are preferred, we set the maximum to today for sampling purposes. upper = today; } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/CQTable.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/CQTable.java index 409a5b3fce2..eb9e57be1a4 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/CQTable.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/CQTable.java @@ -1,11 +1,11 @@ package com.bakdata.conquery.apiv1.query.concept.filter; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; import java.util.Arrays; import java.util.Collections; import java.util.List; import javax.annotation.CheckForNull; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.models.datasets.Column; @@ -60,7 +60,10 @@ public boolean isValidityDateForConnector() { @JsonIgnore @ValidationMethod(message = "Not all Selects belong to Connector.") public boolean isAllSelectsForConnector() { - return selects.stream().map(ConnectorSelectId::getConnector).allMatch(connectorId -> connectorId.equals(connector)); + return selects.stream() + .map(ConnectorSelectId::getConnector) + .allMatch( + connectorId -> connectorId.equals(connector)); } @JsonIgnore @@ -91,8 +94,9 @@ public ValidityDate findValidityDate() { public boolean hasSelectedSecondaryId(SecondaryIdDescriptionId secondaryId) { final Connector resolvedConnector = connector.resolve(); return Arrays.stream(resolvedConnector.getResolvedTable().getColumns()) - .map(Column::getSecondaryId) - .anyMatch(secondaryId::equals); + .map(Column::getSecondaryId) + .anyMatch( + secondaryId::equals); } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java index 8e597eb8136..7267bbece32 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java @@ -1,9 +1,9 @@ package com.bakdata.conquery.apiv1.query.concept.filter; +import jakarta.validation.constraints.NotNull; import java.math.BigDecimal; import java.util.Set; import javax.annotation.Nonnull; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.apiv1.frontend.FrontendFilterType; import com.bakdata.conquery.io.cps.CPSBase; @@ -85,8 +85,7 @@ public String toString() { if (size > 20) { valueString = size + " values"; - } - else { + } else { valueString = readValue().toString(); } @@ -110,8 +109,7 @@ public String toString() { if (size > 20) { valueString = size + " values"; - } - else { + } else { valueString = readValue().toString(); } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/ValidityDateContainer.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/ValidityDateContainer.java index 14617b22788..4598c53f6b6 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/ValidityDateContainer.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/ValidityDateContainer.java @@ -6,7 +6,7 @@ import lombok.RequiredArgsConstructor; @Data -@RequiredArgsConstructor(onConstructor_ = @JsonCreator ) +@RequiredArgsConstructor(onConstructor_ = @JsonCreator) public class ValidityDateContainer { private final ValidityDateId value; } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java index cf9577ff15f..5b0b930933f 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.apiv1.query.concept.specific; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -7,8 +9,6 @@ import java.util.Optional; import java.util.Set; import java.util.function.Consumer; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotEmpty; import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; import com.bakdata.conquery.apiv1.query.CQElement; @@ -154,7 +154,11 @@ public String userLabel(Locale locale) { public String defaultLabel(Locale locale) { // This forces the default label on children even if there was a user label CQElementC10n localized = C10nCache.getLocalized(CQElementC10n.class, locale); - return QueryUtils.createDefaultMultiLabel(children, " " + localized.and() + " ", " " + localized.exists(), locale); + return QueryUtils.createDefaultMultiLabel( + children, + " " + localized.and() + " ", + " " + localized.exists(), + locale); } @Override @@ -175,8 +179,7 @@ public RequiredEntities collectRequiredEntities(QueryExecutionContext context) { if (current == null) { current = next; - } - else { + } else { current = current.intersect(next); } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java index fb9756aa233..322b35ae8b8 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java @@ -1,5 +1,11 @@ package com.bakdata.conquery.apiv1.query.concept.specific; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; +import java.util.*; +import java.util.stream.Collectors; + import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; @@ -28,18 +34,12 @@ import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.fasterxml.jackson.annotation.*; import io.dropwizard.validation.ValidationMethod; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; import lombok.ToString; import lombok.extern.slf4j.Slf4j; -import java.util.*; -import java.util.stream.Collectors; - @Getter @Setter @CPSType(id = "CONCEPT", base = CQElement.class) @@ -77,13 +77,11 @@ public static CQConcept forSelect(SelectId selectId) { switch (selectId) { case ConceptSelectId conceptSelectId -> { - cqConcept.setTables(conceptSelectId.getConcept().resolve() - .getConnectors().stream() - .map(conn -> { - final CQTable table = new CQTable(); - table.setConnector(conn.getId()); - return table; - }).toList()); + cqConcept.setTables(conceptSelectId.getConcept().resolve().getConnectors().stream().map(conn -> { + final CQTable table = new CQTable(); + table.setConnector(conn.getId()); + return table; + }).toList()); cqConcept.setSelects(List.of(conceptSelectId)); } @@ -120,10 +118,11 @@ public static CQConcept forConnector(ConnectorId source) { private static List> createAggregators(ConceptQueryPlan plan, List selects) { return selects.stream() - .map(SelectId::resolve) - .map(Select::createAggregator) - .peek(plan::registerAggregator) - .collect(Collectors.toList()); + .map(SelectId::resolve) + .map(Select::createAggregator) + .peek( + plan::registerAggregator) + .collect(Collectors.toList()); } private static List> createFilters(CQTable table, boolean disableAggregationFilters) { @@ -183,7 +182,8 @@ public Concept getConcept() { @Override public void resolve(QueryResolveContext context) { - aggregateEventDates = !(excludeFromTimeAggregation || DateAggregationMode.NONE.equals(context.getDateAggregationMode())); + aggregateEventDates = !(excludeFromTimeAggregation || DateAggregationMode.NONE.equals( + context.getDateAggregationMode())); tables.forEach(t -> t.resolve(context)); } @@ -216,28 +216,29 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { // Exists aggregators hold a reference to their parent FiltersNode, so they need to be treated separately. // They also don't need aggregation as they simply imitate their reference. - final List existsAggregators = - connectorAggregators.stream() - .filter(ExistsAggregator.class::isInstance) - .map(ExistsAggregator.class::cast) - .toList(); + final List existsAggregators = connectorAggregators.stream() + .filter( + ExistsAggregator.class::isInstance) + .map(ExistsAggregator.class::cast) + .toList(); aggregators.addAll(connectorAggregators); aggregators.removeIf(ExistsAggregator.class::isInstance); - final EventDateUnionAggregator eventDateUnionAggregator = - aggregateEventDates ? new EventDateUnionAggregator() - : null; + final EventDateUnionAggregator eventDateUnionAggregator = aggregateEventDates ? new EventDateUnionAggregator() : null; if (aggregateEventDates) { aggregators.add(eventDateUnionAggregator); } - final QPNode - conceptSpecificNode = - getConcept().createConceptQuery(context, filters, aggregators, eventDateUnionAggregator, selectValidityDate(table)); + final QPNode conceptSpecificNode = getConcept().createConceptQuery( + context, + filters, + aggregators, + eventDateUnionAggregator, + selectValidityDate(table)); if (eventDateUnionAggregator != null) { eventDateUnionAggregator.setOwner(conceptSpecificNode); @@ -247,26 +248,31 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { existsAggregators.forEach(agg -> agg.setReference(conceptSpecificNode)); // Select if matching secondaryId available - final boolean hasSelectedSecondaryId = context.getSelectedSecondaryId() != null && table.hasSelectedSecondaryId(context.getSelectedSecondaryId().getId()); + final boolean hasSelectedSecondaryId = context.getSelectedSecondaryId() != null && table + .hasSelectedSecondaryId( + context.getSelectedSecondaryId().getId()); final ConceptNode node = new ConceptNode( - conceptSpecificNode, - elements.stream().>map(ConceptElementId::resolve).toList(), - table, - // if the node is excluded, don't pass it into the Node. - !excludeFromSecondaryId && hasSelectedSecondaryId ? context.getSelectedSecondaryId() : null + conceptSpecificNode, + elements.stream().>map(ConceptElementId::resolve).toList(), + table, + // if the node is excluded, don't pass it into the Node. + !excludeFromSecondaryId && hasSelectedSecondaryId ? context.getSelectedSecondaryId() : null ); tableNodes.add(node); } // We always merge on concept level - final QPNode outNode = OrNode.of(tableNodes, aggregateEventDates ? DateAggregationAction.MERGE : DateAggregationAction.BLOCK); + final QPNode outNode = OrNode.of( + tableNodes, + aggregateEventDates ? DateAggregationAction.MERGE : DateAggregationAction.BLOCK); // Link concept-level Exists-select to outer node. conceptAggregators.stream() - .filter(aggregator -> aggregator instanceof ExistsAggregator) - .forEach(aggregator -> ((ExistsAggregator) aggregator).setReference(outNode)); + .filter(aggregator -> aggregator instanceof ExistsAggregator) + .forEach( + aggregator -> ((ExistsAggregator) aggregator).setReference(outNode)); return outNode; } @@ -313,9 +319,12 @@ public List getResultInfos() { public RequiredEntities collectRequiredEntities(QueryExecutionContext context) { final Set connectors = getTables().stream().map(CQTable::getConnector).collect(Collectors.toSet()); - return new RequiredEntities(context.getBucketManager() - .getEntitiesWithConcepts(getElements(), - connectors, context.getDateRestriction() + return new RequiredEntities( + context.getBucketManager() + .getEntitiesWithConcepts( + getElements(), + connectors, + context.getDateRestriction() )); } @@ -363,9 +372,7 @@ public void collectNamespacedObjects(Set> iden @Override public void setDefaultSelects() { - final boolean allTablesEmpty = getTables().stream() - .map(CQTable::getSelects) - .allMatch(List::isEmpty); + final boolean allTablesEmpty = getTables().stream().map(CQTable::getSelects).allMatch(List::isEmpty); if (!(getSelects().isEmpty() && (tables.isEmpty() || allTablesEmpty))) { // Don't fill if there are any selects on concept level or on any table level @@ -379,10 +386,14 @@ public void setDefaultSelects() { for (CQTable t : getTables()) { final List conSelects = new ArrayList<>(t.getSelects()); - conSelects.addAll(t.getConnector().resolve() - .getDefaultSelects().stream() + conSelects.addAll( + t.getConnector() + .resolve() + .getDefaultSelects() + .stream() .map(Select::getId) - .map(ConnectorSelectId.class::cast) + .map( + ConnectorSelectId.class::cast) .toList()); t.setSelects(conSelects); diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQDateRestriction.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQDateRestriction.java index af2ffde3fd2..7e6796354ed 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQDateRestriction.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQDateRestriction.java @@ -1,13 +1,13 @@ package com.bakdata.conquery.apiv1.query.concept.specific; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; import java.time.LocalDate; import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Set; import java.util.function.Consumer; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.io.cps.CPSType; @@ -44,8 +44,8 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { QPNode childQueryPlan = child.createQueryPlan(context.withDateRestriction(CDateRange.of(dateRange)), plan); return new DateRestrictingNode( - CDateSet.create(Collections.singleton(CDateRange.of(dateRange))), - childQueryPlan + CDateSet.create(Collections.singleton(CDateRange.of(dateRange))), + childQueryPlan ); } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQNegation.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQNegation.java index 190b021d508..35b5d6dd995 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQNegation.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQNegation.java @@ -1,10 +1,10 @@ package com.bakdata.conquery.apiv1.query.concept.specific; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; import java.util.List; import java.util.Set; import java.util.function.Consumer; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.io.cps.CPSType; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java index cb019ee3072..0ad5fb01223 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.apiv1.query.concept.specific; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -7,8 +9,6 @@ import java.util.Optional; import java.util.Set; import java.util.function.Consumer; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotEmpty; import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; import com.bakdata.conquery.apiv1.query.CQElement; @@ -160,7 +160,11 @@ public String userLabel(Locale locale) { public String defaultLabel(Locale locale) { // This forces the default label on children even if there was a user label CQElementC10n localized = C10nCache.getLocalized(CQElementC10n.class, locale); - return QueryUtils.createDefaultMultiLabel(children, " " + localized.or() + " ", " " + localized.exists(), locale); + return QueryUtils.createDefaultMultiLabel( + children, + " " + localized.or() + " ", + " " + localized.exists(), + locale); } @Override @@ -181,8 +185,7 @@ public RequiredEntities collectRequiredEntities(QueryExecutionContext context) { if (current == null) { current = next; - } - else { + } else { current = current.union(next); } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQReusedQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQReusedQuery.java index 496fa511205..238038de216 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQReusedQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQReusedQuery.java @@ -1,9 +1,9 @@ package com.bakdata.conquery.apiv1.query.concept.specific; +import jakarta.validation.Valid; import java.util.List; import java.util.Set; import java.util.function.Consumer; - import javax.annotation.Nullable; import com.bakdata.conquery.apiv1.query.CQElement; @@ -25,17 +25,17 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonView; -import jakarta.validation.Valid; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @CPSType(id = "SAVED_QUERY", base = CQElement.class) @NoArgsConstructor(onConstructor_ = @JsonCreator) -@Getter @Setter +@Getter +@Setter public class CQReusedQuery extends CQElement { - public CQReusedQuery(ManagedExecutionId executionId){ + public CQReusedQuery(ManagedExecutionId executionId) { this.queryId = executionId; } @@ -68,8 +68,7 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { context = context.withSelectedSecondaryId(null); } - return resolvedQuery.getReusableComponents() - .createQueryPlan(context, plan); + return resolvedQuery.getReusableComponents().createQueryPlan(context, plan); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQTemporal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQTemporal.java index 2b1c725aeee..a7dc3541c76 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQTemporal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQTemporal.java @@ -122,14 +122,15 @@ public final QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan p final List>> shimAggregators = createShimAggregators(nShimAggregators); ConceptQueryPlan indexPlan = createIndexPlan(context); - TemporalQueryNode queryNode = new TemporalQueryNode(context.getStorage().getDataset().getAllIdsTable(), - indexPlan, - getIndexSelector(), - getMode(), - createCompareQueryPlan(context, true), - createCompareQueryPlan(context, false), - getCompareSelector(), - shimAggregators + TemporalQueryNode queryNode = new TemporalQueryNode( + context.getStorage().getDataset().getAllIdsTable(), + indexPlan, + getIndexSelector(), + getMode(), + createCompareQueryPlan(context, true), + createCompareQueryPlan(context, false), + getCompareSelector(), + shimAggregators ); indexPlan.getAggregators().stream().skip(1).forEach(plan::registerAggregator); @@ -144,8 +145,9 @@ public final QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan p private static List>> createShimAggregators(int nShimAggregators) { return IntStream.range(0, nShimAggregators) - .mapToObj(ignored -> new ConstantValueAggregator>(new ArrayList<>())) - .toList(); + .mapToObj( + ignored -> new ConstantValueAggregator>(new ArrayList<>())) + .toList(); } private ConceptQueryPlan createIndexPlan(QueryPlanContext context) { @@ -153,7 +155,8 @@ private ConceptQueryPlan createIndexPlan(QueryPlanContext context) { } private ConceptQueryPlan createCompareQueryPlan(QueryPlanContext context, boolean withAggregation) { - return new ConceptQuery(compare).createQueryPlan(context.withDisableAggregationFilters(withAggregation).withDisableAggregationFilters(withAggregation)); + return new ConceptQuery(compare).createQueryPlan( + context.withDisableAggregationFilters(withAggregation).withDisableAggregationFilters(withAggregation)); } @Override @@ -184,17 +187,22 @@ public List getResultInfos() { resultInfos.addAll(index.getResultInfos()); if (showCompareDate) { - resultInfos.add(new FixedLabelResultInfo(new ResultType.ListT<>(ResultType.Primitive.DATE_RANGE), Collections.emptySet()) { - @Override - public String userColumnName(PrintSettings printSettings) { - return C10N.get(ResultHeadersC10n.class, printSettings.getLocale()).temporalCompareLabel(compare.userLabel(printSettings.getLocale())); - } - - @Override - public ResultSetProcessor.Reader createReader(ResultSetProcessor resultSetProcessor) { - return resultSetProcessor::getDateRangeList; - } - }); + resultInfos.add( + new FixedLabelResultInfo( + new ResultType.ListT<>(ResultType.Primitive.DATE_RANGE), + Collections.emptySet()) { + @Override + public String userColumnName(PrintSettings printSettings) { + return C10N.get(ResultHeadersC10n.class, printSettings.getLocale()) + .temporalCompareLabel( + compare.userLabel(printSettings.getLocale())); + } + + @Override + public ResultSetProcessor.Reader createReader(ResultSetProcessor resultSetProcessor) { + return resultSetProcessor::getDateRangeList; + } + }); } for (ResultInfo resultInfo : compare.getResultInfos()) { @@ -207,8 +215,7 @@ public ResultSetProcessor.Reader createReader(ResultSetProcessor resultSetPro @Override public RequiredEntities collectRequiredEntities(QueryExecutionContext context) { - return index.collectRequiredEntities(context) - .intersect(compare.collectRequiredEntities(context)); + return index.collectRequiredEntities(context).intersect(compare.collectRequiredEntities(context)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java index 3a3b619cdd6..834ce4439d2 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.apiv1.query.concept.specific.external; +import jakarta.validation.constraints.NotEmpty; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -34,7 +35,6 @@ import com.fasterxml.jackson.annotation.JsonView; import com.google.common.collect.Streams; import io.dropwizard.validation.ValidationMethod; -import jakarta.validation.constraints.NotEmpty; import lombok.AccessLevel; import lombok.Getter; import lombok.NoArgsConstructor; @@ -107,12 +107,10 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { } final String[] extraHeaders = Streams.zip( - Arrays.stream(headers), - format.stream(), - (header, format) -> format.equals(EntityResolverUtil.FORMAT_EXTRA) ? header : null - ) - .filter(Objects::nonNull) - .toArray(String[]::new); + Arrays.stream(headers), + format.stream(), + (header, format) -> format.equals(EntityResolverUtil.FORMAT_EXTRA) ? header : null + ).filter(Objects::nonNull).toArray(String[]::new); if (onlySingles) { return createExternalNodeOnlySingle(context, plan, extraHeaders); @@ -126,20 +124,27 @@ public void collectRequiredQueries(Set requiredQueries) { } - private ExternalNode createExternalNodeOnlySingle(QueryPlanContext context, ConceptQueryPlan plan, String[] extraHeaders) { + private ExternalNode createExternalNodeOnlySingle( + QueryPlanContext context, + ConceptQueryPlan plan, + String[] extraHeaders) { // Remove zero element Lists and substitute one element Lists by containing String - final Map> extraFlat = extra.entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - entityToRowMap -> entityToRowMap.getValue().entrySet().stream() - .filter(headerToValue -> !headerToValue.getValue() - .isEmpty()) - .collect(Collectors.toMap( - Map.Entry::getKey, - headerToValue -> headerToValue.getValue() - .get(0) - )) - )); + final Map> extraFlat = extra.entrySet() + .stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + entityToRowMap -> entityToRowMap.getValue() + .entrySet() + .stream() + .filter( + headerToValue -> !headerToValue.getValue().isEmpty()) + .collect( + Collectors.toMap( + Map.Entry::getKey, + headerToValue -> headerToValue.getValue().get(0) + )) + )); final Map> extraAggregators = new HashMap<>(extraHeaders.length); for (String extraHeader : extraHeaders) { @@ -150,10 +155,18 @@ private ExternalNode createExternalNodeOnlySingle(QueryPlanContext conte } - return new ExternalNode<>(context.getStorage().getDataset().getAllIdsTable(), valuesResolved, extraFlat, extraHeaders, extraAggregators); + return new ExternalNode<>( + context.getStorage().getDataset().getAllIdsTable(), + valuesResolved, + extraFlat, + extraHeaders, + extraAggregators); } - private ExternalNode> createExternalNodeForList(QueryPlanContext context, ConceptQueryPlan plan, String[] extraHeaders) { + private ExternalNode> createExternalNodeForList( + QueryPlanContext context, + ConceptQueryPlan plan, + String[] extraHeaders) { final Map>> extraAggregators = new HashMap<>(extraHeaders.length); for (String extraHeader : extraHeaders) { // Just allocating, the result type is irrelevant here @@ -163,11 +176,11 @@ private ExternalNode> createExternalNodeForList(QueryPlanContext co } return new ExternalNode<>( - context.getStorage().getDataset().getAllIdsTable(), - valuesResolved, - extra, - extraHeaders, - extraAggregators + context.getStorage().getDataset().getAllIdsTable(), + valuesResolved, + extra, + extraHeaders, + extraAggregators ); } @@ -175,14 +188,16 @@ private ExternalNode> createExternalNodeForList(QueryPlanContext co public void resolve(QueryResolveContext context) { headers = values[0]; - final EntityResolver.ResolveStatistic resolved = context.getNamespace().getEntityResolver().resolveEntities( + final EntityResolver.ResolveStatistic resolved = context.getNamespace() + .getEntityResolver() + .resolveEntities( values, format, context.getNamespace().getStorage().getIdMapping(), context.getConfig().getIdColumns(), context.getConfig().getLocale().getDateReader(), onlySingles - ); + ); if (resolved.getResolved().isEmpty()) { throw new ConqueryError.ExternalResolveEmptyError(); @@ -190,17 +205,17 @@ public void resolve(QueryResolveContext context) { if (!resolved.getUnreadableDate().isEmpty()) { log.warn( - "Could not read {} dates. Not resolved: {}", - resolved.getUnreadableDate().size(), - resolved.getUnreadableDate().subList(0, Math.min(resolved.getUnreadableDate().size(), 10)) + "Could not read {} dates. Not resolved: {}", + resolved.getUnreadableDate().size(), + resolved.getUnreadableDate().subList(0, Math.min(resolved.getUnreadableDate().size(), 10)) ); } if (!resolved.getUnresolvedId().isEmpty()) { log.warn( - "Could not resolve {} ids. Not resolved: {}", - resolved.getUnresolvedId().size(), - resolved.getUnresolvedId().subList(0, Math.min(resolved.getUnresolvedId().size(), 10)) + "Could not resolve {} ids. Not resolved: {}", + resolved.getUnresolvedId().size(), + resolved.getUnresolvedId().subList(0, Math.min(resolved.getUnresolvedId().size(), 10)) ); } @@ -226,7 +241,8 @@ public List getResultInfos() { final String column = headers[col]; - final ResultType type = onlySingles ? ResultType.Primitive.STRING : new ResultType.ListT<>(ResultType.Primitive.STRING); + final ResultType type = onlySingles ? ResultType.Primitive.STRING : new ResultType.ListT<>( + ResultType.Primitive.STRING); resultInfos.add(new ExternalResultInfo(column, type)); } @@ -272,8 +288,9 @@ public List>> getExtrasForId(String id) { // we need to bring the extras in the correct order List>> inOrder = new ArrayList<>(); Arrays.stream(headers) - .filter(extras::containsKey) - .forEach(header -> inOrder.add(Map.entry(header, extras.get(header)))); + .filter(extras::containsKey) + .forEach( + header -> inOrder.add(Map.entry(header, extras.get(header)))); return inOrder; } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java index 4deffc64a26..5abd4c3d2c0 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java @@ -20,7 +20,7 @@ public void readDates(String value, DateReader dateReader, CDateSet out) { public void readDates(String value, DateReader dateReader, CDateSet out) { final CDateRange parsed = CDateRange.atMost(dateReader.parseToLocalDate(value)); - if(out.isEmpty()){ + if (out.isEmpty()) { out.add(parsed); return; } @@ -37,7 +37,7 @@ public void readDates(String value, DateReader dateReader, CDateSet out) { public void readDates(String value, DateReader dateReader, CDateSet out) { final CDateRange parsed = CDateRange.atLeast(dateReader.parseToLocalDate(value)); - if(out.isEmpty()){ + if (out.isEmpty()) { out.add(parsed); return; } @@ -59,7 +59,7 @@ public void readDates(String value, DateReader dateReader, CDateSet out) { @Override public void readDates(String value, DateReader dateReader, CDateSet out) { CDateSet parsed = dateReader.parseToCDateSet(value); - if (parsed == null ) { + if (parsed == null) { return; } out.addAll(parsed); @@ -77,6 +77,9 @@ public void readDates(String value, DateReader dateReader, CDateSet out) { */ public abstract void readDates(String value, DateReader dateReader, CDateSet out); - public static final Set NAMES = Arrays.stream(DateFormat.values()).map(Enum::name).collect(Collectors.toSet()); + public static final Set NAMES = Arrays.stream(DateFormat.values()) + .map(Enum::name) + .collect( + Collectors.toSet()); } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolver.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolver.java index c93f1e33751..4f93e67ebb3 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolver.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolver.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.apiv1.query.concept.specific.external; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; import java.util.Collections; import java.util.List; import java.util.Map; @@ -9,8 +11,6 @@ import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.util.DateReader; import com.fasterxml.jackson.annotation.JsonIgnore; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; import lombok.Data; public interface EntityResolver { @@ -19,12 +19,12 @@ public interface EntityResolver { * Helper method to try and resolve entities in values using the specified format. */ ResolveStatistic resolveEntities( - @NotEmpty String[][] values, - @NotEmpty List format, - EntityIdMap mapping, - IdColumnConfig idColumnConfig, - @NotNull DateReader dateReader, - boolean onlySingles + @NotEmpty String[][] values, + @NotEmpty List format, + EntityIdMap mapping, + IdColumnConfig idColumnConfig, + @NotNull DateReader dateReader, + boolean onlySingles ); @Data @@ -43,7 +43,11 @@ class ResolveStatistic { private final List unresolvedId; public static ResolveStatistic forEmptyReaders(String[][] values) { - return new ResolveStatistic(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyList(), List.of(values)); + return new ResolveStatistic( + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyList(), + List.of(values)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolverUtil.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolverUtil.java index b26e3a15f55..7f9fc6fef97 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolverUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolverUtil.java @@ -31,9 +31,9 @@ public static CDateSet[] readDates(String[][] values, List format, DateR final CDateSet[] out = new CDateSet[values.length]; final List dateFormats = format.stream() - .map(EntityResolverUtil::resolveDateFormat) - // Don't use Stream#toList to preserve null-values - .collect(Collectors.toList()); + .map(EntityResolverUtil::resolveDateFormat) + // Don't use Stream#toList to preserve null-values + .collect(Collectors.toList()); /* If no format is provided, put empty dates into output. @@ -63,7 +63,7 @@ public static CDateSet[] readDates(String[][] values, List format, DateR String value = values[row][col]; if (StringUtils.isBlank(value)) { - log.trace("Found blank/null value in {}/{} (row/col)", row,col); + log.trace("Found blank/null value in {}/{} (row/col)", row, col); continue; } @@ -81,8 +81,7 @@ public static CDateSet[] readDates(String[][] values, List format, DateR } out[row].addAll(dates); - } - catch (Exception e) { + } catch (Exception e) { // If a value is not parsable, it is included in the exceptions cause message (see DateReader) log.trace("Failed to parse Date in row {}", row, e); // This catch causes `out[row]` to remain `null` which later flags this line as: unresolvedDate @@ -92,25 +91,35 @@ public static CDateSet[] readDates(String[][] values, List format, DateR return out; } - public static void collectExtraData(Map[] extraDataByRow, int rowNum, Map>> extraDataByEntity, String resolvedId) { + public static void collectExtraData( + Map[] extraDataByRow, + int rowNum, + Map>> extraDataByEntity, + String resolvedId) { if (extraDataByRow[rowNum] != null) { for (Map.Entry entry : extraDataByRow[rowNum].entrySet()) { extraDataByEntity.computeIfAbsent(resolvedId, (ignored) -> new HashMap<>()) - .computeIfAbsent(entry.getKey(), (ignored) -> new ArrayList<>()) - .add(entry.getValue()); + .computeIfAbsent( + entry.getKey(), + (ignored) -> new ArrayList<>()) + .add(entry.getValue()); } } } - public static void verifyOnlySingles(boolean onlySingles, Map>> extraDataByEntity) { + public static void verifyOnlySingles( + boolean onlySingles, + Map>> extraDataByEntity) { if (!onlySingles) { return; } // Check that there is at most one value per entity and per column - final boolean alright = extraDataByEntity.values().stream() - .map(Map::values) - .flatMap(Collection::stream) - .allMatch(l -> l.size() <= 1); + final boolean alright = extraDataByEntity.values() + .stream() + .map(Map::values) + .flatMap( + Collection::stream) + .allMatch(l -> l.size() <= 1); if (!alright) { throw new ConqueryError.ExternalResolveOnePerRowError(); } @@ -154,8 +163,7 @@ public static String tryResolveId(String[] row, List[] readExtras(String[][] values, List format) { final String[] names = values[0]; - @SuppressWarnings("unchecked") - final Map[] extrasByRow = new Map[values.length]; + @SuppressWarnings("unchecked") final Map[] extrasByRow = new Map[values.length]; for (int line = 1; line < values.length; line++) { @@ -183,8 +191,7 @@ public static Map[] readExtras(String[][] values, List f private static DateFormat resolveDateFormat(String name) { try { return DateFormat.valueOf(name); - } - catch (IllegalArgumentException e) { + } catch (IllegalArgumentException e) { return null; // Does not exist } } diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ConqueryCommand.java b/backend/src/main/java/com/bakdata/conquery/commands/ConqueryCommand.java index f53bac275b3..6e9ffa423fc 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/ConqueryCommand.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/ConqueryCommand.java @@ -1,72 +1,77 @@ -package com.bakdata.conquery.commands; - -import com.bakdata.conquery.models.config.ConqueryConfig; -import io.dropwizard.core.cli.ConfiguredCommand; -import io.dropwizard.core.setup.Bootstrap; -import io.dropwizard.core.setup.Environment; -import lombok.extern.slf4j.Slf4j; -import net.sourceforge.argparse4j.inf.Namespace; -import org.eclipse.jetty.util.component.ContainerLifeCycle; - -@Slf4j -public abstract class ConqueryCommand extends ConfiguredCommand { - - /** - * Creates a new environment command. - * - * @param name the name of the command, used for command line invocation - */ - protected ConqueryCommand(String name, String description) { - super(name, description); - } - - @Override - protected void run(Bootstrap bootstrap, Namespace namespace, ConqueryConfig configuration) throws Exception { - final Environment environment = new Environment(bootstrap.getApplication().getName(), - bootstrap.getObjectMapper(), - bootstrap.getValidatorFactory(), - bootstrap.getMetricRegistry(), - bootstrap.getClassLoader(), - bootstrap.getHealthCheckRegistry(), - configuration); - configuration.getMetricsFactory().configure(environment.lifecycle(), bootstrap.getMetricRegistry()); - configuration.getServerFactory().configure(environment); - - bootstrap.run(configuration, environment); - - ContainerLifeCycle lifeCycle = new ContainerLifeCycle(); - try { - run(environment, namespace, configuration); - environment.lifecycle().attach(lifeCycle); - lifeCycle.start(); - - Runtime.getRuntime().addShutdownHook(new Thread() { - @Override - public void run() { - try { - lifeCycle.stop(); - } - catch (Exception e) { - log.error("Interrupted during shutdown", e); - } - } - }); - } - catch(Throwable t) { - log.error("Uncaught Exception in "+getName(), t); - lifeCycle.stop(); - throw t; - } - } - - /** - * Runs the command with the given {@link Environment} and {@link ConqueryConfig}. - * - * @param environment the configured environment - * @param namespace the parsed command line namespace - * @param configuration the configuration object - * @throws Exception if something goes wrong - */ - protected abstract void run(Environment environment, Namespace namespace, ConqueryConfig configuration) throws Exception; - -} +package com.bakdata.conquery.commands; + +import com.bakdata.conquery.models.config.ConqueryConfig; +import io.dropwizard.core.cli.ConfiguredCommand; +import io.dropwizard.core.setup.Bootstrap; +import io.dropwizard.core.setup.Environment; +import lombok.extern.slf4j.Slf4j; +import net.sourceforge.argparse4j.inf.Namespace; +import org.eclipse.jetty.util.component.ContainerLifeCycle; + +@Slf4j +public abstract class ConqueryCommand extends ConfiguredCommand { + + /** + * Creates a new environment command. + * + * @param name the name of the command, used for command line invocation + */ + protected ConqueryCommand(String name, String description) { + super(name, description); + } + + @Override + protected void run( + Bootstrap bootstrap, + Namespace namespace, + ConqueryConfig configuration) throws Exception { + final Environment environment = new Environment( + bootstrap.getApplication().getName(), + bootstrap.getObjectMapper(), + bootstrap.getValidatorFactory(), + bootstrap.getMetricRegistry(), + bootstrap.getClassLoader(), + bootstrap.getHealthCheckRegistry(), + configuration); + configuration.getMetricsFactory().configure(environment.lifecycle(), bootstrap.getMetricRegistry()); + configuration.getServerFactory().configure(environment); + + bootstrap.run(configuration, environment); + + ContainerLifeCycle lifeCycle = new ContainerLifeCycle(); + try { + run(environment, namespace, configuration); + environment.lifecycle().attach(lifeCycle); + lifeCycle.start(); + + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + try { + lifeCycle.stop(); + } catch (Exception e) { + log.error("Interrupted during shutdown", e); + } + } + }); + } catch (Throwable t) { + log.error("Uncaught Exception in " + getName(), t); + lifeCycle.stop(); + throw t; + } + } + + /** + * Runs the command with the given {@link Environment} and {@link ConqueryConfig}. + * + * @param environment the configured environment + * @param namespace the parsed command line namespace + * @param configuration the configuration object + * @throws Exception if something goes wrong + */ + protected abstract void run( + Environment environment, + Namespace namespace, + ConqueryConfig configuration) throws Exception; + +} diff --git a/backend/src/main/java/com/bakdata/conquery/commands/DistributedStandaloneCommand.java b/backend/src/main/java/com/bakdata/conquery/commands/DistributedStandaloneCommand.java index 6b5a9b12ddc..f29b4d9053b 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/DistributedStandaloneCommand.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/DistributedStandaloneCommand.java @@ -30,7 +30,10 @@ public class DistributedStandaloneCommand extends ServerCommand private Clock clock = Clock.systemDefaultZone(); public DistributedStandaloneCommand() { - super(new NoOpConquery(), "standalone", "starts a manager node and shard node(s) at the same time in a single JVM."); + super( + new NoOpConquery(), + "standalone", + "starts a manager node and shard node(s) at the same time in a single JVM."); } @Override @@ -41,7 +44,8 @@ protected void run(Environment environment, Namespace namespace, ConqueryConfig if (configuration.getStorage() instanceof XodusStoreFactory) { final Path managerDir = ((XodusStoreFactory) configuration.getStorage()).getDirectory().resolve("manager"); - managerConfig = configuration.withStorage(((XodusStoreFactory) configuration.getStorage()).withDirectory(managerDir)); + managerConfig = configuration.withStorage( + ((XodusStoreFactory) configuration.getStorage()).withDirectory(managerDir)); } manager = new ClusterManagerProvider().provideManager(managerConfig, environment); @@ -60,8 +64,11 @@ protected void run(Environment environment, Namespace namespace, ConqueryConfig ConqueryConfig clone = configuration; if (configuration.getStorage() instanceof XodusStoreFactory) { - final Path managerDir = ((XodusStoreFactory) configuration.getStorage()).getDirectory().resolve("shard-node" + id); - clone = configuration.withStorage(((XodusStoreFactory) configuration.getStorage()).withDirectory(managerDir)); + final Path managerDir = ((XodusStoreFactory) configuration.getStorage()).getDirectory() + .resolve( + "shard-node" + id); + clone = configuration.withStorage( + ((XodusStoreFactory) configuration.getStorage()).withDirectory(managerDir)); } sc.run(clone, environment); diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java index c49933870b5..b22e51838fc 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.commands; +import jakarta.validation.Validator; import java.time.Duration; import java.util.ArrayList; import java.util.Collection; @@ -10,7 +11,6 @@ import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import jakarta.validation.Validator; import com.bakdata.conquery.io.cps.CPSTypeIdResolver; import com.bakdata.conquery.io.jersey.RESTServer; @@ -92,9 +92,7 @@ public void run(Manager manager) throws InterruptedException { configureApiServlet(config, environment); - maintenanceService = environment.lifecycle() - .scheduledExecutorService("Maintenance Service") - .build(); + maintenanceService = environment.lifecycle().scheduledExecutorService("Maintenance Service").build(); environment.lifecycle().manage(this); @@ -108,13 +106,13 @@ public void run(Manager manager) throws InterruptedException { admin.register(); log.info("Registering ResourcesProvider"); - for (Class resourceProvider : CPSTypeIdResolver.listImplementations(ResourcesProvider.class)) { + for (Class resourceProvider : CPSTypeIdResolver.listImplementations( + ResourcesProvider.class)) { try { ResourcesProvider provider = resourceProvider.getConstructor().newInstance(); provider.registerResources(this); providers.add(provider); - } - catch (Exception e) { + } catch (Exception e) { log.error("Failed to register Resource {}", resourceProvider, e); } } @@ -135,7 +133,10 @@ protected void configure() { } }); - getInternalMapperFactory().customizeApiObjectMapper(environment.getObjectMapper(), getDatasetRegistry(), getMetaStorage()); + getInternalMapperFactory().customizeApiObjectMapper( + environment.getObjectMapper(), + getDatasetRegistry(), + getMetaStorage()); } @@ -147,25 +148,28 @@ public void loadNamespaces() throws InterruptedException { } static void loadNamespaces( - Collection namespaceStorages, - DatasetRegistry registry, - MetaStorage metaStorage, - Environment environment) throws InterruptedException { + Collection namespaceStorages, + DatasetRegistry registry, + MetaStorage metaStorage, + Environment environment) throws InterruptedException { - try(ExecutorService loaders = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors())) { + try (ExecutorService loaders = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors())) { final List loadTasks = new ArrayList<>(namespaceStorages.size()); // Namespaces load their storage themselves, so they can inject Namespace relevant objects into stored objects for (NamespaceStorage namespaceStorage : namespaceStorages) { - Future future = loaders.submit(() -> registry.createNamespace(namespaceStorage, metaStorage, environment)); + Future future = loaders.submit( + () -> registry.createNamespace(namespaceStorage, metaStorage, environment)); loadTasks.add(new NamespaceLoadTask(namespaceStorage, future)); } loaders.shutdown(); while (!loaders.awaitTermination(1, TimeUnit.MINUTES)) { final int countLoaded = registry.getNamespaces().size(); - log.debug("Waiting for dataset namespaces to load. {} are already finished. {} pending.", countLoaded, namespaceStorages.size() - - countLoaded); + log.debug( + "Waiting for dataset namespaces to load. {} are already finished. {} pending.", + countLoaded, + namespaceStorages.size() - countLoaded); } // Check for failed storages @@ -173,14 +177,13 @@ static void loadNamespaces( for (NamespaceLoadTask loadTask : loadTasks) { try { loadTask.future().get(); - } - catch (ExecutionException exception) { + } catch (ExecutionException exception) { final Throwable cause = exception.getCause(); final String storagePath = loadTask.storage().getPathName(); log.error( - "Failed to load persisted dataset namespace [storage={}]. The dataset is unavailable; application startup will be aborted.", - storagePath, - cause + "Failed to load persisted dataset namespace [storage={}]. The dataset is unavailable; application startup will be aborted.", + storagePath, + cause ); failures.add(new NamespaceLoadFailure(storagePath, cause)); } @@ -193,12 +196,16 @@ static void loadNamespaces( } } - private static IllegalStateException createNamespaceLoadingException(int namespaceCount, List failures) { + private static IllegalStateException createNamespaceLoadingException( + int namespaceCount, + List failures) { final List failedStoragePaths = failures.stream().map(NamespaceLoadFailure::storagePath).toList(); final IllegalStateException exception = new IllegalStateException( - "Failed to load %d of %d persisted dataset namespaces: %s" - .formatted(failures.size(), namespaceCount, failedStoragePaths), - failures.getFirst().cause() + "Failed to load %d of %d persisted dataset namespaces: %s".formatted( + failures.size(), + namespaceCount, + failedStoragePaths), + failures.getFirst().cause() ); failures.stream().skip(1).map(NamespaceLoadFailure::cause).forEach(exception::addSuppressed); @@ -213,7 +220,8 @@ private record NamespaceLoadFailure(String storagePath, Throwable cause) { private void loadMetaStorage() { log.info("Opening MetaStorage"); - getMetaStorage().openStores(getInternalMapperFactory().createManagerPersistenceMapper(getDatasetRegistry(), getMetaStorage())); + getMetaStorage().openStores( + getInternalMapperFactory().createManagerPersistenceMapper(getDatasetRegistry(), getMetaStorage())); getMetaStorage().loadKeys(); @@ -226,11 +234,14 @@ private void loadMetaStorage() { private void registerTasks(Manager manager, Environment environment, ConqueryConfig config) { environment.admin().addTask(formScanner); - environment.admin().addTask( - new QueryCleanupTask(getMetaStorage(), Duration.of( + environment.admin() + .addTask( + new QueryCleanupTask( + getMetaStorage(), + Duration.of( config.getQueries().getOldQueriesTime().getQuantity(), config.getQueries().getOldQueriesTime().getUnit().toChronoUnit() - ))); + ))); environment.admin().addTask(new PermissionCleanupTask(getMetaStorage())); manager.getAdminTasks().forEach(environment.admin()::addTask); @@ -255,16 +266,14 @@ public void stop() throws Exception { for (ResourcesProvider provider : providers) { try { provider.close(); - } - catch (Exception e) { + } catch (Exception e) { log.error("{} could not be closed", provider, e); } } try { getMetaStorage().close(); - } - catch (Exception e) { + } catch (Exception e) { log.error("{} could not be closed", getMetaStorage(), e); } diff --git a/backend/src/main/java/com/bakdata/conquery/commands/MigrateCommand.java b/backend/src/main/java/com/bakdata/conquery/commands/MigrateCommand.java index c09221bc907..cefa141c0dc 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/MigrateCommand.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/MigrateCommand.java @@ -71,39 +71,39 @@ public MigrateCommand() { @Override public void configure(Subparser subparser) { - subparser - .addArgument("--in") - .help("Input storage directory.") - .required(true) - .type(Arguments.fileType().verifyIsDirectory().verifyCanRead()); - - subparser - .addArgument("--out") - .help("Output storage directory.") - .required(true) - .type(Arguments.fileType()); - - subparser - .addArgument("--in-gzip") - .help("If true, values are ungzipped before deserialization.") - .setDefault(true) - .type(Arguments.booleanType()); - - subparser - .addArgument("--out-gzip") - .help("If true, values are gzipped before writing.") - .setDefault(true) - .type(Arguments.booleanType()); - - subparser - .addArgument("--script") - .help("Migration Script returning a closure implementing MigrationScriptFactory. See supplementary example.groovy for details.\nSignature: String env, String store, String key, ObjectNode value -> return new Tuple(key,value)") - .required(true) - .type(Arguments.fileType().verifyCanRead().verifyCanExecute()); + subparser.addArgument("--in") + .help("Input storage directory.") + .required(true) + .type( + Arguments.fileType().verifyIsDirectory().verifyCanRead()); + + subparser.addArgument("--out").help("Output storage directory.").required(true).type(Arguments.fileType()); + + subparser.addArgument("--in-gzip") + .help("If true, values are ungzipped before deserialization.") + .setDefault( + true) + .type(Arguments.booleanType()); + + subparser.addArgument("--out-gzip") + .help("If true, values are gzipped before writing.") + .setDefault(true) + .type( + Arguments.booleanType()); + + subparser.addArgument("--script") + .help( + "Migration Script returning a closure implementing MigrationScriptFactory. See supplementary example.groovy for details.\nSignature: String env, String store, String key, ObjectNode value -> return new Tuple(key,value)") + .required( + true) + .type(Arguments.fileType().verifyCanRead().verifyCanExecute()); } @Override - protected void run(io.dropwizard.core.setup.Environment environment, Namespace namespace, ConqueryConfig configuration) throws Exception { + protected void run( + io.dropwizard.core.setup.Environment environment, + Namespace namespace, + ConqueryConfig configuration) throws Exception { final File inStoreDirectory = namespace.get("in"); final File outStoreDirectory = namespace.get("out"); @@ -133,43 +133,43 @@ protected void run(io.dropwizard.core.setup.Environment environment, Namespace n final ObjectMapper mapper = Jackson.BINARY_MAPPER; - Arrays.stream(environments) - .parallel() - .forEach(xenv -> - { - final File environmentDirectory = new File(outStoreDirectory, xenv.getName()); - if (environmentDirectory.exists()) { - throw new IllegalArgumentException("File or folder already exists. Cannot create environment: " + environmentDirectory.getAbsolutePath()); - } - if (!environmentDirectory.mkdirs()) { - throw new IllegalArgumentException("Cannot create environment directory: " + environmentDirectory.getAbsolutePath()); - } - - processEnvironment(xenv, logsize, environmentDirectory, migrator, mapper, inGzip, outGzip); - }); + Arrays.stream(environments).parallel().forEach(xenv -> { + final File environmentDirectory = new File(outStoreDirectory, xenv.getName()); + if (environmentDirectory.exists()) { + throw new IllegalArgumentException( + "File or folder already exists. Cannot create environment: " + environmentDirectory + .getAbsolutePath()); + } + if (!environmentDirectory.mkdirs()) { + throw new IllegalArgumentException( + "Cannot create environment directory: " + environmentDirectory.getAbsolutePath()); + } + + processEnvironment(xenv, logsize, environmentDirectory, migrator, mapper, inGzip, outGzip); + }); } private void processEnvironment( - File inStoreDirectory, - long logSize, - File outStoreDirectory, - Function4> migrator, - ObjectMapper mapper, - boolean inGzip, - boolean outGzip) { + File inStoreDirectory, + long logSize, + File outStoreDirectory, + Function4> migrator, + ObjectMapper mapper, + boolean inGzip, + boolean outGzip) { final jetbrains.exodus.env.Environment inEnvironment; try { inEnvironment = Environments.newInstance( - inStoreDirectory, - new EnvironmentConfig().setLogFileSize(logSize) - .setEnvIsReadonly(true) - .setEnvCompactOnOpen(false) - .setEnvCloseForcedly(true) - .setGcEnabled(false) + inStoreDirectory, + new EnvironmentConfig().setLogFileSize(logSize) + .setEnvIsReadonly(true) + .setEnvCompactOnOpen( + false) + .setEnvCloseForcedly(true) + .setGcEnabled(false) ); - } - catch (Exception e) { + } catch (Exception e) { throw new IllegalArgumentException("Unable to open store in " + inStoreDirectory, e); } @@ -177,12 +177,10 @@ private void processEnvironment( try { // we dump first, then enable GC. outEnvironment = Environments.newInstance( - outStoreDirectory, - new EnvironmentConfig().setLogFileSize(logSize) - .setGcEnabled(false) + outStoreDirectory, + new EnvironmentConfig().setLogFileSize(logSize).setGcEnabled(false) ); - } - catch (Exception e) { + } catch (Exception e) { throw new IllegalArgumentException("Unable to open store in " + outStoreDirectory, e); } @@ -194,16 +192,16 @@ private void processEnvironment( outEnvironment.clear(); for (String store : stores) { - final Store inStore = - inEnvironment.computeInExclusiveTransaction(tx -> inEnvironment.openStore(store, StoreConfig.USE_EXISTING, tx, false)); + final Store inStore = inEnvironment.computeInExclusiveTransaction( + tx -> inEnvironment.openStore(store, StoreConfig.USE_EXISTING, tx, false)); if (inStore == null) { log.error("{} does not exist, aborting.", store); continue; } - final Store outStore = - outEnvironment.computeInExclusiveTransaction(tx -> outEnvironment.openStore(store, StoreConfig.WITHOUT_DUPLICATES_WITH_PREFIXING, tx, true)); + final Store outStore = outEnvironment.computeInExclusiveTransaction( + tx -> outEnvironment.openStore(store, StoreConfig.WITHOUT_DUPLICATES_WITH_PREFIXING, tx, true)); if (outEnvironment.computeInReadonlyTransaction(outStore::count) > 0) { log.warn("Store is not empty, aborting."); @@ -227,12 +225,12 @@ private void processEnvironment( } private void migrateStore( - Store inStore, - Store outStore, - Function4> migrator, - ObjectMapper mapper, - boolean inGzip, - boolean outGzip) { + Store inStore, + Store outStore, + Function4> migrator, + ObjectMapper mapper, + boolean inGzip, + boolean outGzip) { final Environment inEnvironment = inStore.getEnvironment(); final Environment outEnvironment = outStore.getEnvironment(); @@ -273,17 +271,18 @@ private void migrateStore( outStore.put(writeTx, new ArrayByteIterable(keyWritten), new ArrayByteIterable(valueWritten)); if (++processed % (1 + (count / 10)) == 0) { - log.info("Processed {} / {} ({}%)", processed, count, Math.round(100f * (float) processed / (float) count)); + log.info( + "Processed {} / {} ({}%)", + processed, + count, + Math.round(100f * (float) processed / (float) count)); } } - } - catch (JsonMappingException e) { + } catch (JsonMappingException e) { log.error("Failed to Map", e); - } - catch (IOException e) { + } catch (IOException e) { log.error("Failed in IO", e); - } - finally { + } finally { readTx.abort(); } diff --git a/backend/src/main/java/com/bakdata/conquery/commands/PreprocessorCommand.java b/backend/src/main/java/com/bakdata/conquery/commands/PreprocessorCommand.java index eb9621b5ccc..bca99cec906 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/PreprocessorCommand.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/PreprocessorCommand.java @@ -1,358 +1,392 @@ -package com.bakdata.conquery.commands; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.function.Predicate; -import java.util.zip.GZIPInputStream; -import jakarta.validation.ValidationException; -import jakarta.validation.Validator; - -import com.bakdata.conquery.ConqueryConstants; -import com.bakdata.conquery.io.jackson.Jackson; -import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.exceptions.ValidatorHelper; -import com.bakdata.conquery.models.preproc.PreprocessedHeader; -import com.bakdata.conquery.models.preproc.PreprocessedReader; -import com.bakdata.conquery.models.preproc.PreprocessingJob; -import com.bakdata.conquery.models.preproc.Preprocessor; -import com.bakdata.conquery.models.preproc.TableImportDescriptor; -import com.bakdata.conquery.models.preproc.TableInputDescriptor; -import com.bakdata.conquery.util.io.ConqueryMDC; -import com.bakdata.conquery.util.io.LogUtil; -import com.bakdata.conquery.util.io.ProgressBar; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.dropwizard.core.setup.Environment; -import lombok.SneakyThrows; -import lombok.experimental.FieldNameConstants; -import lombok.extern.slf4j.Slf4j; -import net.sourceforge.argparse4j.impl.Arguments; -import net.sourceforge.argparse4j.impl.type.BooleanArgumentType; -import net.sourceforge.argparse4j.impl.type.FileArgumentType; -import net.sourceforge.argparse4j.impl.type.StringArgumentType; -import net.sourceforge.argparse4j.inf.ArgumentGroup; -import net.sourceforge.argparse4j.inf.Namespace; -import net.sourceforge.argparse4j.inf.Subparser; -import org.apache.commons.io.FileUtils; -import org.jetbrains.annotations.NotNull; - -@Slf4j -@FieldNameConstants -public class PreprocessorCommand extends ConqueryCommand { - - private final List failed = Collections.synchronizedList(new ArrayList<>()); - private final List success = Collections.synchronizedList(new ArrayList<>()); - private ExecutorService pool; - private boolean isFailFast; - private boolean isStrict = true; - - public PreprocessorCommand() { - this(null); - } - - public PreprocessorCommand(ExecutorService pool) { - super("preprocess", "Preprocesses all the files in the given input directories. This has to be done only if the model or the files changed."); - this.pool = pool; - } - - @SneakyThrows - public static boolean requiresProcessing(PreprocessingJob preprocessingJob) { - ConqueryMDC.setLocation(preprocessingJob.toString()); - if (preprocessingJob.getPreprocessedFile().exists()) { - - log.info("EXISTS ALREADY"); - - final int currentHash = preprocessingJob.getDescriptor() - .calculateValidityHash(preprocessingJob.getCsvDirectory(), preprocessingJob.getTag()); - - - final ObjectMapper om = Jackson.BINARY_MAPPER.copy(); - try (final PreprocessedReader parser = new PreprocessedReader(new GZIPInputStream(new FileInputStream(preprocessingJob.getPreprocessedFile())), om)) { - - final PreprocessedHeader header = parser.readHeader(); - - if (header.getValidityHash() == currentHash) { - log.info("\tHASH STILL VALID"); - return false; - } - log.info("\tHASH OUTDATED"); - } - catch (Exception e) { - log.error("\tHEADER READING FAILED", e); - return false; - } - } - else { - log.info("DOES NOT EXIST"); - } - - return true; - } - - /** - * Configure CLI based configuration for preprocessor. - */ - @Override - public void configure(Subparser subparser) { - super.configure(subparser); - - final ArgumentGroup group = subparser.addArgumentGroup("Preprocessing CLI Config") - .description("Optional arguments to do a single import step by hand. Overrides json configuration."); - - group.addArgument("--in") - .required(true) - .type(new FileArgumentType().verifyIsDirectory().verifyCanRead()) - .help("Directory containing the input files (in csv or gzipped csv format)."); - - group.addArgument("--out") - .required(false) - .type(new FileArgumentType().verifyIsDirectory().verifyCanCreate().verifyCanWrite()) - .help("Directory to write the output cqpp files to."); - - group.addArgument("--desc") - .required(true) - .type(new FileArgumentType().verifyCanRead()) - .nargs("*") - .help("Directory containing the import description files (*.import.json)."); - - group.addArgument("--tag") - .required(false) - .type(new StringArgumentType()) - .nargs("*") - .help("Optional tags for input and output files: Will change input files from `filename.csv.gz` to `filename.$tag.csv.gz` and output files from `filename.cqpp` to `filename.$tag.cqpp`. Tag will also override the import-id to tag."); - - group.addArgument("--fast-fail") - .action(Arguments.storeTrue()) - .setDefault(false) - .help("Stop preprocessing and exit with failure if an error occurs that prevents the generation of a cqpp."); - - group.addArgument("--strict") - .type(new BooleanArgumentType()) - .setDefault(true) - .help("Escalate missing files to errors."); - - group.addArgument("--buckets") - .type(Integer.class) - .setDefault(100) - .help("Number of buckets to use for id-hashing. This value is required to be a constant per-dataset."); - - } - - @Override - protected void run(Environment environment, Namespace namespace, ConqueryConfig config) throws Exception { - if (pool == null) { - pool = Executors.newFixedThreadPool(config.getPreprocessor().getNThreads()); - } - - // Tag if present is appended to input-file csvs, output-file cqpp and used as id of cqpps - - // Seems to be a bug with dropwizard and boolean default-values - isFailFast = Optional.ofNullable(namespace.getBoolean("fast-fail")).orElse(false); - isStrict = Optional.ofNullable(namespace.getBoolean("strict")).orElse(false); - - final List tags = namespace.getList("tag"); - - final File inDir = namespace.get("in"); - final File outDir = namespace.get("out"); - final List descriptionFilesRoot = namespace.getList("desc"); - final int buckets = namespace.getInt("buckets"); - - - log.info("Preprocessing from command line config."); - - final Collection jobs = collectJobs(descriptionFilesRoot, tags, inDir, outDir, environment); - - final List broken = validateJobs(jobs, environment); - - jobs.removeIf(Predicate.not(PreprocessorCommand::requiresProcessing)); - - preprocessJobs(jobs, buckets, config); - - - log.info("Successfully Preprocess {} Jobs:", success.size()); - success.forEach(desc -> log.info("\tSucceeded Preprocessing for {}", desc)); - - if (!broken.isEmpty()) { - log.warn("Did not find {} Files", broken.size()); - broken.forEach(desc -> log.warn("\tDid not find file for {}", desc)); - } - - if (isFailed()) { - log.error("Failed {} Preprocessing Jobs:", failed.size()); - failed.forEach(desc -> log.error("\tFailed Preprocessing for {}", desc)); - doFail(); - } - } - - @NotNull - private List validateJobs(Collection jobs, Environment environment) { - final List broken = new ArrayList<>(); - - for (final Iterator iterator = jobs.iterator(); iterator.hasNext(); ) { - final PreprocessingJob job = iterator.next(); - - try { - ValidatorHelper.failOnError(log, environment.getValidator().validate(job.getDescriptor())); - } - catch (ValidationException exception) { - log.error("Descriptor {} is not valid.", job.getDescriptor(), exception); - broken.add(job); - iterator.remove(); - continue; - } - - for (TableInputDescriptor input : job.getDescriptor().getInputs()) { - - final File sourceFile = Preprocessor.resolveSourceFile(input.getSourceFile(), job.getCsvDirectory(), job.getTag()); - if (!sourceFile.exists()) { - log.error("Did not find file `{}` for Preprocessing[{}].", sourceFile, job); - broken.add(job); - iterator.remove(); - } - } - } - - // This will halt preprocessing immediately. - if (isStrict && !broken.isEmpty()) { - log.error("FAILED Preprocessing, files are missing or invalid."); - doFail(); - } - return broken; - } - - @NotNull - private Collection collectJobs(List descriptionFiles, List tags, File inDir, File outDir, Environment environment) - throws IOException { - final Collection jobs = new ArrayList<>(); - - if (tags == null || tags.isEmpty()) { - for (File desc : descriptionFiles) { - final List descriptions = - findPreprocessingDescriptions(desc, inDir, outDir, Optional.empty(), environment.getValidator()); - jobs.addAll(descriptions); - } - } - else { - for (String tag : tags) { - for (File desc : descriptionFiles) { - final List jobDescriptions = - findPreprocessingDescriptions(desc, inDir, outDir, Optional.of(tag), environment.getValidator()); - - jobs.addAll(jobDescriptions); - } - } - } - return jobs; - } - - private void preprocessJobs(Collection jobs, int buckets, ConqueryConfig config) throws InterruptedException { - final long totalSize = jobs.stream() - .mapToLong(PreprocessingJob::estimateTotalCsvSizeBytes) - .sum(); - - log.info("Required to preprocess {} in total", FileUtils.byteCountToDisplaySize(totalSize)); - - final ProgressBar totalProgress = new ProgressBar(totalSize); - - for (PreprocessingJob job : jobs) { - pool.submit(() -> { - ConqueryMDC.setLocation(job.toString()); - try { - Preprocessor.preprocess(job, totalProgress, config, buckets); - success.add(job.toString()); - } - catch (FileNotFoundException e) { - log.warn("Did not find file `{}` for preprocessing.", e.getMessage()); - addMissing(job); - } - catch (Exception e) { - log.error("Failed to preprocess " + LogUtil.printPath(job.getDescriptionFile()), e); - addFailed(job); - } - }); - } - - pool.shutdown(); - pool.awaitTermination(24, TimeUnit.HOURS); - - ConqueryMDC.clearLocation(); - } - - private void addMissing(PreprocessingJob job) { - log.error("Files are missing for {}", job); - if (isStrict) { - addFailed(job); - } - } - - private void addFailed(PreprocessingJob job) { - if (isFailFast) { - doFail(); - } - failed.add(job.toString()); - } - - public List findPreprocessingDescriptions(File descriptionFiles, File inDir, File outputDir, Optional tag, Validator validator) - throws IOException { - final List out = new ArrayList<>(); - - final File[] files = descriptionFiles.isFile() - ? new File[]{descriptionFiles} - : descriptionFiles.listFiles(((dir, name) -> name.endsWith(ConqueryConstants.EXTENSION_DESCRIPTION))); - - if (files == null) { - return Collections.emptyList(); - } - - for (File descriptionFile : files) { - tryExtractDescriptor(validator, tag, descriptionFile, outputDir, inDir) - .ifPresent(out::add); - } - return out; - } - - private boolean isFailed() { - return !failed.isEmpty(); - } - - private Optional tryExtractDescriptor(Validator validator, Optional tag, File descriptionFile, File outputDir, File csvDir) { - try { - final TableImportDescriptor - descriptor = - TableImportDescriptor.read(descriptionFile); - - validator.validate(validator); - - final PreprocessingJob preprocessingJob = new PreprocessingJob(csvDir.toPath(), descriptionFile, outputDir.toPath(), tag, descriptor); - - - // Override name to tag if present - tag.ifPresent(descriptor::setName); - - return Optional.of(preprocessingJob); - } - catch (Exception e) { - log.error("Failed to process " + LogUtil.printPath(descriptionFile), e); - if (isFailFast) { - doFail(); - } - failed.add(descriptionFile.toString()); - } - return Optional.empty(); - } - - private void doFail() { - System.exit(1); - } - -} +package com.bakdata.conquery.commands; + +import jakarta.validation.ValidationException; +import jakarta.validation.Validator; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; +import java.util.zip.GZIPInputStream; + +import com.bakdata.conquery.ConqueryConstants; +import com.bakdata.conquery.io.jackson.Jackson; +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.exceptions.ValidatorHelper; +import com.bakdata.conquery.models.preproc.PreprocessedHeader; +import com.bakdata.conquery.models.preproc.PreprocessedReader; +import com.bakdata.conquery.models.preproc.PreprocessingJob; +import com.bakdata.conquery.models.preproc.Preprocessor; +import com.bakdata.conquery.models.preproc.TableImportDescriptor; +import com.bakdata.conquery.models.preproc.TableInputDescriptor; +import com.bakdata.conquery.util.io.ConqueryMDC; +import com.bakdata.conquery.util.io.LogUtil; +import com.bakdata.conquery.util.io.ProgressBar; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.dropwizard.core.setup.Environment; +import lombok.SneakyThrows; +import lombok.experimental.FieldNameConstants; +import lombok.extern.slf4j.Slf4j; +import net.sourceforge.argparse4j.impl.Arguments; +import net.sourceforge.argparse4j.impl.type.BooleanArgumentType; +import net.sourceforge.argparse4j.impl.type.FileArgumentType; +import net.sourceforge.argparse4j.impl.type.StringArgumentType; +import net.sourceforge.argparse4j.inf.ArgumentGroup; +import net.sourceforge.argparse4j.inf.Namespace; +import net.sourceforge.argparse4j.inf.Subparser; +import org.apache.commons.io.FileUtils; +import org.jetbrains.annotations.NotNull; + +@Slf4j +@FieldNameConstants +public class PreprocessorCommand extends ConqueryCommand { + + private final List failed = Collections.synchronizedList(new ArrayList<>()); + private final List success = Collections.synchronizedList(new ArrayList<>()); + private ExecutorService pool; + private boolean isFailFast; + private boolean isStrict = true; + + public PreprocessorCommand() { + this(null); + } + + public PreprocessorCommand(ExecutorService pool) { + super( + "preprocess", + "Preprocesses all the files in the given input directories. This has to be done only if the model or the files changed."); + this.pool = pool; + } + + @SneakyThrows + public static boolean requiresProcessing(PreprocessingJob preprocessingJob) { + ConqueryMDC.setLocation(preprocessingJob.toString()); + if (preprocessingJob.getPreprocessedFile().exists()) { + + log.info("EXISTS ALREADY"); + + final int currentHash = preprocessingJob.getDescriptor() + .calculateValidityHash( + preprocessingJob.getCsvDirectory(), + preprocessingJob.getTag()); + + + final ObjectMapper om = Jackson.BINARY_MAPPER.copy(); + try (final PreprocessedReader parser = new PreprocessedReader( + new GZIPInputStream(new FileInputStream(preprocessingJob.getPreprocessedFile())), + om)) { + + final PreprocessedHeader header = parser.readHeader(); + + if (header.getValidityHash() == currentHash) { + log.info("\tHASH STILL VALID"); + return false; + } + log.info("\tHASH OUTDATED"); + } catch (Exception e) { + log.error("\tHEADER READING FAILED", e); + return false; + } + } else { + log.info("DOES NOT EXIST"); + } + + return true; + } + + /** + * Configure CLI based configuration for preprocessor. + */ + @Override + public void configure(Subparser subparser) { + super.configure(subparser); + + final ArgumentGroup group = subparser.addArgumentGroup("Preprocessing CLI Config") + .description( + "Optional arguments to do a single import step by hand. Overrides json configuration."); + + group.addArgument("--in") + .required(true) + .type(new FileArgumentType().verifyIsDirectory().verifyCanRead()) + .help( + "Directory containing the input files (in csv or gzipped csv format)."); + + group.addArgument("--out") + .required(false) + .type( + new FileArgumentType().verifyIsDirectory().verifyCanCreate().verifyCanWrite()) + .help( + "Directory to write the output cqpp files to."); + + group.addArgument("--desc") + .required(true) + .type(new FileArgumentType().verifyCanRead()) + .nargs("*") + .help( + "Directory containing the import description files (*.import.json)."); + + group.addArgument("--tag") + .required(false) + .type(new StringArgumentType()) + .nargs("*") + .help( + "Optional tags for input and output files: Will change input files from `filename.csv.gz` to `filename.$tag.csv.gz` and output files from `filename.cqpp` to `filename.$tag.cqpp`. Tag will also override the import-id to tag."); + + group.addArgument("--fast-fail") + .action(Arguments.storeTrue()) + .setDefault(false) + .help( + "Stop preprocessing and exit with failure if an error occurs that prevents the generation of a cqpp."); + + group.addArgument("--strict") + .type(new BooleanArgumentType()) + .setDefault(true) + .help( + "Escalate missing files to errors."); + + group.addArgument("--buckets") + .type(Integer.class) + .setDefault(100) + .help( + "Number of buckets to use for id-hashing. This value is required to be a constant per-dataset."); + + } + + @Override + protected void run(Environment environment, Namespace namespace, ConqueryConfig config) throws Exception { + if (pool == null) { + pool = Executors.newFixedThreadPool(config.getPreprocessor().getNThreads()); + } + + // Tag if present is appended to input-file csvs, output-file cqpp and used as id of cqpps + + // Seems to be a bug with dropwizard and boolean default-values + isFailFast = Optional.ofNullable(namespace.getBoolean("fast-fail")).orElse(false); + isStrict = Optional.ofNullable(namespace.getBoolean("strict")).orElse(false); + + final List tags = namespace.getList("tag"); + + final File inDir = namespace.get("in"); + final File outDir = namespace.get("out"); + final List descriptionFilesRoot = namespace.getList("desc"); + final int buckets = namespace.getInt("buckets"); + + + log.info("Preprocessing from command line config."); + + final Collection jobs = collectJobs(descriptionFilesRoot, tags, inDir, outDir, environment); + + final List broken = validateJobs(jobs, environment); + + jobs.removeIf(Predicate.not(PreprocessorCommand::requiresProcessing)); + + preprocessJobs(jobs, buckets, config); + + + log.info("Successfully Preprocess {} Jobs:", success.size()); + success.forEach(desc -> log.info("\tSucceeded Preprocessing for {}", desc)); + + if (!broken.isEmpty()) { + log.warn("Did not find {} Files", broken.size()); + broken.forEach(desc -> log.warn("\tDid not find file for {}", desc)); + } + + if (isFailed()) { + log.error("Failed {} Preprocessing Jobs:", failed.size()); + failed.forEach(desc -> log.error("\tFailed Preprocessing for {}", desc)); + doFail(); + } + } + + @NotNull + private List validateJobs(Collection jobs, Environment environment) { + final List broken = new ArrayList<>(); + + for (final Iterator iterator = jobs.iterator(); iterator.hasNext();) { + final PreprocessingJob job = iterator.next(); + + try { + ValidatorHelper.failOnError(log, environment.getValidator().validate(job.getDescriptor())); + } catch (ValidationException exception) { + log.error("Descriptor {} is not valid.", job.getDescriptor(), exception); + broken.add(job); + iterator.remove(); + continue; + } + + for (TableInputDescriptor input : job.getDescriptor().getInputs()) { + + final File sourceFile = Preprocessor.resolveSourceFile( + input.getSourceFile(), + job.getCsvDirectory(), + job.getTag()); + if (!sourceFile.exists()) { + log.error("Did not find file `{}` for Preprocessing[{}].", sourceFile, job); + broken.add(job); + iterator.remove(); + } + } + } + + // This will halt preprocessing immediately. + if (isStrict && !broken.isEmpty()) { + log.error("FAILED Preprocessing, files are missing or invalid."); + doFail(); + } + return broken; + } + + @NotNull + private Collection collectJobs( + List descriptionFiles, + List tags, + File inDir, + File outDir, + Environment environment) throws IOException { + final Collection jobs = new ArrayList<>(); + + if (tags == null || tags.isEmpty()) { + for (File desc : descriptionFiles) { + final List descriptions = findPreprocessingDescriptions( + desc, + inDir, + outDir, + Optional.empty(), + environment.getValidator()); + jobs.addAll(descriptions); + } + } else { + for (String tag : tags) { + for (File desc : descriptionFiles) { + final List jobDescriptions = findPreprocessingDescriptions( + desc, + inDir, + outDir, + Optional.of(tag), + environment.getValidator()); + + jobs.addAll(jobDescriptions); + } + } + } + return jobs; + } + + private void preprocessJobs( + Collection jobs, + int buckets, + ConqueryConfig config) throws InterruptedException { + final long totalSize = jobs.stream().mapToLong(PreprocessingJob::estimateTotalCsvSizeBytes).sum(); + + log.info("Required to preprocess {} in total", FileUtils.byteCountToDisplaySize(totalSize)); + + final ProgressBar totalProgress = new ProgressBar(totalSize); + + for (PreprocessingJob job : jobs) { + pool.submit(() -> { + ConqueryMDC.setLocation(job.toString()); + try { + Preprocessor.preprocess(job, totalProgress, config, buckets); + success.add(job.toString()); + } catch (FileNotFoundException e) { + log.warn("Did not find file `{}` for preprocessing.", e.getMessage()); + addMissing(job); + } catch (Exception e) { + log.error("Failed to preprocess " + LogUtil.printPath(job.getDescriptionFile()), e); + addFailed(job); + } + }); + } + + pool.shutdown(); + pool.awaitTermination(24, TimeUnit.HOURS); + + ConqueryMDC.clearLocation(); + } + + private void addMissing(PreprocessingJob job) { + log.error("Files are missing for {}", job); + if (isStrict) { + addFailed(job); + } + } + + private void addFailed(PreprocessingJob job) { + if (isFailFast) { + doFail(); + } + failed.add(job.toString()); + } + + public List findPreprocessingDescriptions( + File descriptionFiles, + File inDir, + File outputDir, + Optional tag, + Validator validator) throws IOException { + final List out = new ArrayList<>(); + + final File[] files = descriptionFiles.isFile() ? new File[]{descriptionFiles} : descriptionFiles.listFiles( + ((dir, name) -> name.endsWith(ConqueryConstants.EXTENSION_DESCRIPTION))); + + if (files == null) { + return Collections.emptyList(); + } + + for (File descriptionFile : files) { + tryExtractDescriptor(validator, tag, descriptionFile, outputDir, inDir).ifPresent(out::add); + } + return out; + } + + private boolean isFailed() { + return !failed.isEmpty(); + } + + private Optional tryExtractDescriptor( + Validator validator, + Optional tag, + File descriptionFile, + File outputDir, + File csvDir) { + try { + final TableImportDescriptor descriptor = TableImportDescriptor.read(descriptionFile); + + validator.validate(validator); + + final PreprocessingJob preprocessingJob = new PreprocessingJob( + csvDir.toPath(), + descriptionFile, + outputDir.toPath(), + tag, + descriptor); + + + // Override name to tag if present + tag.ifPresent(descriptor::setName); + + return Optional.of(preprocessingJob); + } catch (Exception e) { + log.error("Failed to process " + LogUtil.printPath(descriptionFile), e); + if (isFailFast) { + doFail(); + } + failed.add(descriptionFile.toString()); + } + return Optional.empty(); + } + + private void doFail() { + System.exit(1); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/commands/RecodeStoreCommand.java b/backend/src/main/java/com/bakdata/conquery/commands/RecodeStoreCommand.java index 816aa8819bc..709214aca9e 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/RecodeStoreCommand.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/RecodeStoreCommand.java @@ -28,27 +28,17 @@ public RecodeStoreCommand() { @Override public void configure(Subparser subparser) { - subparser - .addArgument("--in") - .help("Input store.") - .required(true) - .type(Arguments.fileType().verifyIsDirectory().verifyCanRead()); - - subparser - .addArgument("--out") - .help("Output store.") - .required(true) - .type(Arguments.fileType()); - - subparser - .addArgument("--in_logsize") - .help("Logsize of incoming store.") - .required(true); - - subparser - .addArgument("--out_logsize") - .help("Logsize of outgoing store.") - .required(true); + subparser.addArgument("--in") + .help("Input store.") + .required(true) + .type( + Arguments.fileType().verifyIsDirectory().verifyCanRead()); + + subparser.addArgument("--out").help("Output store.").required(true).type(Arguments.fileType()); + + subparser.addArgument("--in_logsize").help("Logsize of incoming store.").required(true); + + subparser.addArgument("--out_logsize").help("Logsize of outgoing store.").required(true); } @@ -61,40 +51,42 @@ public void run(Bootstrap bootstrap, Namespace namespace) { final File outStoreDirectory = namespace.get("out"); final long outLogSize = DataSize.parse(namespace.get("out_logsize")).toKilobytes(); - log.info("Recoding Storage at `{} ({})` to `{} ({})`", inStoreDirectory, inLogSize, outStoreDirectory, outLogSize); + log.info( + "Recoding Storage at `{} ({})` to `{} ({})`", + inStoreDirectory, + inLogSize, + outStoreDirectory, + outLogSize); final File[] environments = inStoreDirectory.listFiles(File::isDirectory); - if(environments == null){ + if (environments == null) { log.error("In Store is empty"); return; } - Arrays.stream(environments) - .parallel() - .forEach(environment -> - { - final File environmentDirectory = new File(outStoreDirectory, environment.getName()); - environmentDirectory.mkdirs(); - processEnvironment(environment, inLogSize, environmentDirectory, outLogSize); - }); + Arrays.stream(environments).parallel().forEach(environment -> { + final File environmentDirectory = new File(outStoreDirectory, environment.getName()); + environmentDirectory.mkdirs(); + processEnvironment(environment, inLogSize, environmentDirectory, outLogSize); + }); } private void processEnvironment(File inStoreDirectory, long inLogSize, File outStoreDirectory, long outLogSize) { final jetbrains.exodus.env.Environment inEnvironment = Environments.newInstance( - inStoreDirectory, - new EnvironmentConfig().setLogFileSize(inLogSize) - .setEnvIsReadonly(true) - .setEnvCompactOnOpen(false) - .setEnvCloseForcedly(true) - .setGcEnabled(false) + inStoreDirectory, + new EnvironmentConfig().setLogFileSize(inLogSize) + .setEnvIsReadonly(true) + .setEnvCompactOnOpen( + false) + .setEnvCloseForcedly(true) + .setGcEnabled(false) ); // we dump first, then enable GC. final jetbrains.exodus.env.Environment outEnvironment = Environments.newInstance( - outStoreDirectory, - new EnvironmentConfig().setLogFileSize(outLogSize) - .setGcEnabled(false) + outStoreDirectory, + new EnvironmentConfig().setLogFileSize(outLogSize).setGcEnabled(false) ); @@ -105,16 +97,16 @@ private void processEnvironment(File inStoreDirectory, long inLogSize, File outS outEnvironment.clear(); for (String store : stores) { - final Store inStore = - inEnvironment.computeInExclusiveTransaction(tx -> inEnvironment.openStore(store, StoreConfig.USE_EXISTING, tx, false)); + final Store inStore = inEnvironment.computeInExclusiveTransaction( + tx -> inEnvironment.openStore(store, StoreConfig.USE_EXISTING, tx, false)); if (inStore == null) { log.error("{} does not exist, aborting.", store); continue; } - final Store outStore = - outEnvironment.computeInExclusiveTransaction(tx -> outEnvironment.openStore(store, StoreConfig.WITHOUT_DUPLICATES_WITH_PREFIXING, tx, true)); + final Store outStore = outEnvironment.computeInExclusiveTransaction( + tx -> outEnvironment.openStore(store, StoreConfig.WITHOUT_DUPLICATES_WITH_PREFIXING, tx, true)); if (outEnvironment.computeInReadonlyTransaction(outStore::count) > 0) { log.warn("Store is not empty, aborting."); @@ -137,7 +129,11 @@ private void processEnvironment(File inStoreDirectory, long inLogSize, File outS inEnvironment.close(); } - private void doRecode(jetbrains.exodus.env.Environment inEnvironment, Store inStore, jetbrains.exodus.env.Environment outEnvironment, Store outStore) { + private void doRecode( + jetbrains.exodus.env.Environment inEnvironment, + Store inStore, + jetbrains.exodus.env.Environment outEnvironment, + Store outStore) { final Transaction readTx = inEnvironment.beginReadonlyTransaction(); @@ -148,22 +144,25 @@ private void doRecode(jetbrains.exodus.env.Environment inEnvironment, Store inSt log.info("Contains {} Entries", count); - try(final Cursor cursor = inStore.openCursor(readTx)) { + try (final Cursor cursor = inStore.openCursor(readTx)) { while (cursor.getNext()) { outStore.putRight(writeTx, cursor.getKey(), cursor.getValue()); if (++processed % (1 + (count / 10)) == 0) { - log.info("Processed {} / {} ({}%)", processed, count, Math.round(100f * (float) processed / (float) count)); + log.info( + "Processed {} / {} ({}%)", + processed, + count, + Math.round(100f * (float) processed / (float) count)); } } - } - finally { + } finally { readTx.abort(); } log.info("Processed {} / {} ({}%)", processed, count, 100); - if(!writeTx.commit()){ + if (!writeTx.commit()) { log.error("Failed to commit Tx for {}", outEnvironment); } } diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ShardCommand.java b/backend/src/main/java/com/bakdata/conquery/commands/ShardCommand.java index bf52085427f..3d83c2570ff 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/ShardCommand.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/ShardCommand.java @@ -20,7 +20,10 @@ public ShardCommand() { } @Override - protected void run(Bootstrap bootstrap, Namespace namespace, ConqueryConfig configuration) throws Exception { + protected void run( + Bootstrap bootstrap, + Namespace namespace, + ConqueryConfig configuration) throws Exception { bootstrap.addBundle(new ShardNode()); super.run(bootstrap, namespace, configuration); @@ -32,7 +35,7 @@ protected void run(Environment environment, Namespace namespace, ConqueryConfig Clear application connectors for a shard, before building the server, as we only expose the metrics through the admin connector. */ - ((DefaultServerFactory)configuration.getServerFactory()).setApplicationConnectors(Collections.emptyList()); + ((DefaultServerFactory) configuration.getServerFactory()).setApplicationConnectors(Collections.emptyList()); super.run(environment, namespace, configuration); } diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java index 2ae61504cf1..6f409124e71 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java @@ -56,18 +56,17 @@ public void run(ConqueryConfig config, Environment environment) throws Exception InternalMapperFactory internalMapperFactory = new InternalMapperFactory(config, environment.getValidator()); workers = new ShardWorkers( - config.getQueries().getExecutionPool(), - internalMapperFactory, - config.getQueries().getSecondaryIdSubPlanRetention(), - clock + config.getQueries().getExecutionPool(), + internalMapperFactory, + config.getQueries().getSecondaryIdSubPlanRetention(), + clock ); lifecycle.manage(workers); environment.admin().addTask(new LoadStorageTask(getName(), null, workers)); - clusterConnection = - new ClusterConnectionShard(config, environment, workers, internalMapperFactory); + clusterConnection = new ClusterConnectionShard(config, environment, workers, internalMapperFactory); lifecycle.manage(clusterConnection); @@ -80,12 +79,14 @@ public void run(ConqueryConfig config, Environment environment) throws Exception for (WorkerStorage workerStorage : workerStorages) { loaders.submit(() -> { try { - workersDone.add(workers.openWorker(workerStorage, config.isFailOnError(), config.getStorage().isLoadStoresOnStart())); - } - catch (Exception e) { + workersDone.add( + workers.openWorker( + workerStorage, + config.isFailOnError(), + config.getStorage().isLoadStoresOnStart())); + } catch (Exception e) { log.error("Failed reading Storage", e); - } - finally { + } finally { log.debug("DONE reading Storage {}", workerStorage); ConqueryMDC.clearLocation(); } @@ -95,15 +96,16 @@ public void run(ConqueryConfig config, Environment environment) throws Exception loaders.shutdown(); while (!loaders.awaitTermination(1, TimeUnit.MINUTES)) { - log.debug("Waiting for Worker workers to load. {} are already finished. {} pending", workersDone.size(), workerStorages.size() - - workersDone.size()); + log.debug( + "Waiting for Worker workers to load. {} are already finished. {} pending", + workersDone.size(), + workerStorages.size() - workersDone.size()); } log.info("All Worker loaded: {}", workers.getWorkers().size()); } - public boolean isBusy() { return clusterConnection.isBusy() || workers.isBusy(); } diff --git a/backend/src/main/java/com/bakdata/conquery/commands/StandaloneCommand.java b/backend/src/main/java/com/bakdata/conquery/commands/StandaloneCommand.java index 6d720caa8c0..7805b9144e8 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/StandaloneCommand.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/StandaloneCommand.java @@ -17,6 +17,7 @@ public interface StandaloneCommand { Environment getEnvironment(); void setClock(Clock clock); + Clock getClock(); } diff --git a/backend/src/main/java/com/bakdata/conquery/internationalization/CQElementC10n.java b/backend/src/main/java/com/bakdata/conquery/internationalization/CQElementC10n.java index 14ef245ba9e..103a22f8e9f 100644 --- a/backend/src/main/java/com/bakdata/conquery/internationalization/CQElementC10n.java +++ b/backend/src/main/java/com/bakdata/conquery/internationalization/CQElementC10n.java @@ -5,14 +5,14 @@ /** * Cosmopolitan (C10n) internationalization provider for elements of a query description. - * These are used to generate expressive default labels for queries. + * These are used to generate expressive default labels for queries. */ public interface CQElementC10n { - + @En("Uploaded-List") @De("Hochgeladene-Liste") String external(); - + @En("Query") @De("Anfrage") String reused(); diff --git a/backend/src/main/java/com/bakdata/conquery/internationalization/DateContextResolutionC10n.java b/backend/src/main/java/com/bakdata/conquery/internationalization/DateContextResolutionC10n.java index a1202bf5190..afe3d8d769b 100644 --- a/backend/src/main/java/com/bakdata/conquery/internationalization/DateContextResolutionC10n.java +++ b/backend/src/main/java/com/bakdata/conquery/internationalization/DateContextResolutionC10n.java @@ -7,15 +7,15 @@ public interface DateContextResolutionC10n { @En("complete") @De("Gesamt") String complete(); - + @En("year") @De("Jahr") String year(); - + @En("quarter") @De("Quartal") String quarter(); - + @En("day") @De("Tag") String day(); diff --git a/backend/src/main/java/com/bakdata/conquery/internationalization/ExcelSheetNameC10n.java b/backend/src/main/java/com/bakdata/conquery/internationalization/ExcelSheetNameC10n.java index 36dbb161035..45ef76139b3 100644 --- a/backend/src/main/java/com/bakdata/conquery/internationalization/ExcelSheetNameC10n.java +++ b/backend/src/main/java/com/bakdata/conquery/internationalization/ExcelSheetNameC10n.java @@ -4,7 +4,7 @@ import c10n.annotations.En; public interface ExcelSheetNameC10n { - @En("Result") - @De("Ergebnis") - String result(); + @En("Result") + @De("Ergebnis") + String result(); } diff --git a/backend/src/main/java/com/bakdata/conquery/internationalization/ExportFormC10n.java b/backend/src/main/java/com/bakdata/conquery/internationalization/ExportFormC10n.java index 2ba1231cda6..215b3e03f41 100644 --- a/backend/src/main/java/com/bakdata/conquery/internationalization/ExportFormC10n.java +++ b/backend/src/main/java/com/bakdata/conquery/internationalization/ExportFormC10n.java @@ -2,12 +2,13 @@ import c10n.annotations.De; import c10n.annotations.En; + /** * Cosmopolitan (C10n) internationalization provider for the export form. * Used to generate an expressive default name. */ public interface ExportFormC10n { - + @En("Data Export") @De("Datenexport") String getType(); diff --git a/backend/src/main/java/com/bakdata/conquery/internationalization/ResultHeadersC10n.java b/backend/src/main/java/com/bakdata/conquery/internationalization/ResultHeadersC10n.java index b6bebfad10b..7ab1352579a 100644 --- a/backend/src/main/java/com/bakdata/conquery/internationalization/ResultHeadersC10n.java +++ b/backend/src/main/java/com/bakdata/conquery/internationalization/ResultHeadersC10n.java @@ -4,7 +4,7 @@ import c10n.annotations.En; public interface ResultHeadersC10n { - + @En("resolution") @De("Zeiteinheit") String resolution(); @@ -36,7 +36,7 @@ public interface ResultHeadersC10n { @En("feature_date_range") @De("Vorbeobachtungszeitraum") String featureDateRange(); - + @En("outcome_date_range") @De("Nachbeobachtungszeitraum") String outcomeDateRange(); diff --git a/backend/src/main/java/com/bakdata/conquery/io/cps/CPSBase.java b/backend/src/main/java/com/bakdata/conquery/io/cps/CPSBase.java index a429c07994a..652d4574458 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/cps/CPSBase.java +++ b/backend/src/main/java/com/bakdata/conquery/io/cps/CPSBase.java @@ -7,8 +7,10 @@ import com.fasterxml.jackson.annotation.JacksonAnnotationsInside; import com.fasterxml.jackson.databind.annotation.JsonTypeIdResolver; + @Retention(RetentionPolicy.RUNTIME) @Target({ElementType.TYPE}) -@JacksonAnnotationsInside @JsonTypeIdResolver(CPSTypeIdResolver.class) +@JacksonAnnotationsInside +@JsonTypeIdResolver(CPSTypeIdResolver.class) public @interface CPSBase { -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/io/cps/CPSMap.java b/backend/src/main/java/com/bakdata/conquery/io/cps/CPSMap.java index 5d3b6884a20..c1777f6c105 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/cps/CPSMap.java +++ b/backend/src/main/java/com/bakdata/conquery/io/cps/CPSMap.java @@ -14,7 +14,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public class CPSMap implements Iterable, String>>{ +public class CPSMap implements Iterable, String>> { private Multimap, String> class2TypeLabels = HashMultimap.create(); private Map> typeLabel2Class; @@ -25,21 +25,21 @@ public void add(String id, Class type) { public void calculateInverse() { ImmutableMultimap, String> immutable = ImmutableMultimap.copyOf(class2TypeLabels); class2TypeLabels = immutable; - + ImmutableMultimap> rev = immutable.inverse(); int failed = 0; - for(Entry>> e:rev.asMap().entrySet()) { - if(e.getValue().size()>1) { + for (Entry>> e : rev.asMap().entrySet()) { + if (e.getValue().size() > 1) { log.error("There are multiple objects with the type id {}: {}", e.getKey(), e.getValue()); failed++; } } - if(failed > 0) { - throw new IllegalStateException(failed+" errors (see above)"); + if (failed > 0) { + throw new IllegalStateException(failed + " errors (see above)"); } typeLabel2Class = rev.entries().stream().collect(Collectors.toMap(Entry::getKey, Entry::getValue)); } - + public void merge(CPSMap other) { class2TypeLabels.putAll(other.class2TypeLabels); } @@ -59,11 +59,11 @@ public Set> getClasses() { public String getTypeIdForClass(Class suggestedType) { Collection ids = class2TypeLabels.get(suggestedType); - if(ids.isEmpty()) { + if (ids.isEmpty()) { //check if other base CPSType anno = suggestedType.getAnnotation(CPSType.class); - if(anno == null) { - throw new IllegalStateException("There is no id for the class "+suggestedType+" for."); + if (anno == null) { + throw new IllegalStateException("There is no id for the class " + suggestedType + " for."); } return anno.id(); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/cps/CPSType.java b/backend/src/main/java/com/bakdata/conquery/io/cps/CPSType.java index a5fc2eb9200..975506b2556 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/cps/CPSType.java +++ b/backend/src/main/java/com/bakdata/conquery/io/cps/CPSType.java @@ -11,10 +11,12 @@ @Repeatable(CPSTypes.class) public @interface CPSType { String id(); + Class base(); + /** - * When true, it indicates that the class uses sub-typing information in its id, which is saved in objects. if so, the class must implement + * When true, it indicates that the class uses sub-typing information in its id, which is saved in objects. if so, the class must implement * {@link SubTyped}. The consumed and produced type ids used by serdes (Jackson) are of the form <TYPE-ID>{@link CPSTypeIdResolver#SEPARATOR_SUB_TYPE}<SUBTYPE-ID>. */ boolean subTyped() default false; -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/io/cps/CPSTypeIdResolver.java b/backend/src/main/java/com/bakdata/conquery/io/cps/CPSTypeIdResolver.java index a25666ccd1e..92f6355f8e4 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/cps/CPSTypeIdResolver.java +++ b/backend/src/main/java/com/bakdata/conquery/io/cps/CPSTypeIdResolver.java @@ -47,13 +47,13 @@ public void init(JavaType baseType) { //this creates an aggregate map of all the children Iterable> types = Traverser.forGraph( - (SuccessorsFunction>) node -> { - Class superclass = node.getSuperclass(); - List> interfaces = Arrays.asList(node.getInterfaces()); - return superclass == null - ? interfaces - : Iterables.concat(interfaces, Collections.singleton(superclass)); - } + (SuccessorsFunction>) node -> { + Class superclass = node.getSuperclass(); + List> interfaces = Arrays.asList(node.getInterfaces()); + return superclass == null ? interfaces : Iterables.concat( + interfaces, + Collections.singleton(superclass)); + } ).breadthFirst(baseType.getRawClass()); for (Class type : types) { @@ -70,15 +70,14 @@ public void init(JavaType baseType) { log.info("Scanning Classpath"); //scan classpaths for annotated child classes - SCAN_RESULT = new ClassGraph() - .enableClassInfo() - .enableAnnotationInfo() - //TODO if ever necessary, use an environment variable also - .acceptPackages( - "com.bakdata", - "tests" - ) - .scan(); + SCAN_RESULT = new ClassGraph().enableClassInfo() + .enableAnnotationInfo() + //TODO if ever necessary, use an environment variable also + .acceptPackages( + "com.bakdata", + "tests" + ) + .scan(); log.info("Scanned: {} classes in classpath", SCAN_RESULT.getAllClasses().size()); Set> types = new HashSet<>(); @@ -94,16 +93,19 @@ public void init(JavaType baseType) { //check if base is marked as base CPSBase baseAnno = anno.base().getAnnotation(CPSBase.class); if (baseAnno == null) { - throw new IllegalStateException("The class " + anno.base() + " is used as a CPSBase in " + type + " but not annotated as such."); + throw new IllegalStateException( + "The class " + anno + .base() + " is used as a CPSBase in " + type + " but not annotated as such."); } if (!anno.base().isAssignableFrom(type)) { - throw new IllegalStateException("The class " + anno.base() + " is used as a CPSBase in " + type + " but type is no subclass of it."); + throw new IllegalStateException( + "The class " + anno + .base() + " is used as a CPSBase in " + type + " but type is no subclass of it."); } if (anno.subTyped() && !SubTyped.class.isAssignableFrom(type)) { - throw new IllegalStateException("The class " - + type - + " is flagged to support a subtyping information but does not implement " - + SubTyped.class.getName()); + throw new IllegalStateException( + "The class " + type + " is flagged to support a subtyping information but does not implement " + SubTyped.class + .getName()); } map.add(anno.id(), type); @@ -115,8 +117,7 @@ public void init(JavaType baseType) { CPSMap map = globalMap.get(b); if (map == null) { log.warn("\tBase Class {}:\tNo registered types", b); - } - else { + } else { log.info("\tBase Class {}", b.getSimpleName()); map.calculateInverse(); for (Entry, String> e : map) { @@ -131,7 +132,11 @@ public void init(JavaType baseType) { public JavaType typeFromId(DatabindContext context, String id) throws IOException { Class result = cpsMap.getClassFromId(truncateSubTypeInformation(id)); if (result == null) { - throw new InvalidTypeIdException(null, "There is no type " + id + " for " + baseType.getTypeName() + ". Try: " + getDescForKnownTypeIds(), baseType, id); + throw new InvalidTypeIdException( + null, + "There is no type " + id + " for " + baseType.getTypeName() + ". Try: " + getDescForKnownTypeIds(), + baseType, + id); } String subTypeInfo = extractSubTypeInformation(id); if (!Strings.isNullOrEmpty(subTypeInfo)) { @@ -180,13 +185,11 @@ public static Class getImplementation(Class base, public static Set, Class>> listImplementations() { return globalMap.entrySet() - .stream() - ., Class>>flatMap(e -> e.getValue() - .getClasses() - .stream() - .map(v -> Pair.of(e.getKey(), v)) - ) - .collect(Collectors.toSet()); + .stream() + ., Class>>flatMap( + e -> e.getValue().getClasses().stream().map(v -> Pair.of(e.getKey(), v)) + ) + .collect(Collectors.toSet()); } @Override @@ -196,7 +199,8 @@ public String idFromValueAndType(Object value, Class suggestedType) { //check if other base CPSType anno = value.getClass().getAnnotation(CPSType.class); if (anno == null) { - throw new IllegalStateException("There is no id for the class " + suggestedType + " for " + baseType.getTypeName() + "."); + throw new IllegalStateException( + "There is no id for the class " + suggestedType + " for " + baseType.getTypeName() + "."); } return anno.id(); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendApi.java b/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendApi.java index 477e5340660..f0c0844d3ca 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendApi.java +++ b/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendApi.java @@ -1,11 +1,5 @@ package com.bakdata.conquery.io.external.form; -import java.net.URI; -import java.net.URL; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.function.Function; import jakarta.ws.rs.client.Client; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.client.Invocation; @@ -13,6 +7,12 @@ import jakarta.ws.rs.core.GenericType; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; +import java.net.URI; +import java.net.URL; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.function.Function; import com.bakdata.conquery.apiv1.forms.ExternalForm; import com.bakdata.conquery.models.auth.entities.User; @@ -49,7 +49,18 @@ public class ExternalFormBackendApi { private final WebTarget baseTarget; private final URL conqueryApiUrl; - public ExternalFormBackendApi(Client client, URI baseURI, String formConfigPath, String postFormPath, String statusTemplatePath, String cancelTaskPath, String healthCheckPath, String versionPath, Function tokenCreator, URL conqueryApiUrl, AuthenticationClientFilterProvider authFilterProvider) { + public ExternalFormBackendApi( + Client client, + URI baseURI, + String formConfigPath, + String postFormPath, + String statusTemplatePath, + String cancelTaskPath, + String healthCheckPath, + String versionPath, + Function tokenCreator, + URL conqueryApiUrl, + AuthenticationClientFilterProvider authFilterProvider) { this.client = client; this.tokenCreator = tokenCreator; @@ -76,9 +87,11 @@ public List getFormConfigs() { log.debug("Getting form configurations from: {}", formConfigTarget); return formConfigTarget.request(MediaType.APPLICATION_JSON_TYPE) - .acceptLanguage(I18n.LOCALE.get()) - .buildGet().invoke(new GenericType<>() { - }); + .acceptLanguage( + I18n.LOCALE.get()) + .buildGet() + .invoke(new GenericType<>() { + }); } public ExternalTaskState postForm(ExternalForm form, User originalUser, User serviceUser, Dataset dataset) { @@ -90,7 +103,9 @@ public ExternalTaskState postForm(ExternalForm form, User originalUser, User ser // If user is not allowed to download, only provide them with statistics. - final DatasetDetail detail = originalUser.isPermitted(dataset, Ability.DOWNLOAD) ? DatasetDetail.FULL : DatasetDetail.STATISTIC; + final DatasetDetail detail = originalUser.isPermitted( + dataset, + Ability.DOWNLOAD) ? DatasetDetail.FULL : DatasetDetail.STATISTIC; webTarget = webTarget.queryParam(QUERY_SCOPE, detail); final Invocation.Builder request = webTarget.request(MediaType.APPLICATION_JSON_TYPE); @@ -100,12 +115,16 @@ public ExternalTaskState postForm(ExternalForm form, User originalUser, User ser final String originalUserToken = tokenCreator.apply(originalUser); request.header(HTTP_HEADER_CQ_API_URL, conqueryApiUrl) - .header(HTTP_HEADER_CQ_AUTHENTICATION, serviceUserToken) - .header(HTTP_HEADER_CQ_AUTHENTICATION_ORIGINAL, originalUserToken) - .acceptLanguage(I18n.LOCALE.get()) - ; - - ExternalTaskState post = request.post(Entity.entity(form.getExternalApiPayload(), MediaType.APPLICATION_JSON_TYPE), ExternalTaskState.class); + .header( + HTTP_HEADER_CQ_AUTHENTICATION, + serviceUserToken) + .header(HTTP_HEADER_CQ_AUTHENTICATION_ORIGINAL, originalUserToken) + .acceptLanguage( + I18n.LOCALE.get()); + + ExternalTaskState post = request.post( + Entity.entity(form.getExternalApiPayload(), MediaType.APPLICATION_JSON_TYPE), + ExternalTaskState.class); return post; } @@ -115,16 +134,15 @@ public ExternalTaskState getFormState(UUID externalId) { log.debug("Getting status from: {}", getStatusTargetResolved); return getStatusTargetResolved.request(MediaType.APPLICATION_JSON_TYPE) - .acceptLanguage(I18n.LOCALE.get()) - .get(ExternalTaskState.class); + .acceptLanguage(I18n.LOCALE.get()) + .get( + ExternalTaskState.class); } public Response getResult(final URI resultURL) { log.debug("Query external form result from {}", resultURL); - return client.target(baseTarget.getUri().resolve(resultURL)).request() - .acceptLanguage(I18n.LOCALE.get()) - .get(); + return client.target(baseTarget.getUri().resolve(resultURL)).request().acceptLanguage(I18n.LOCALE.get()).get(); } @@ -140,8 +158,10 @@ public ExternalTaskState cancelTask(UUID taskId) { log.debug("Cancelling task {}", taskId); final ExternalTaskState taskState = cancelTaskTarget.resolveTemplates(Map.of(TASK_ID, taskId)) - .request() - .post(null, ExternalTaskState.class); + .request() + .post( + null, + ExternalTaskState.class); if (taskState.getStatus() != TaskStatus.CANCELLED) { log.warn("Task `{}` was cancelled, but is still in state {}", taskId, taskState.getStatus()); diff --git a/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalTaskState.java b/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalTaskState.java index fc8deaba1ba..ea2acb70299 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalTaskState.java +++ b/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalTaskState.java @@ -1,13 +1,12 @@ package com.bakdata.conquery.io.external.form; -import java.math.BigDecimal; -import java.util.List; -import java.util.UUID; - import jakarta.validation.Valid; import jakarta.validation.constraints.DecimalMax; import jakarta.validation.constraints.DecimalMin; import jakarta.validation.constraints.NotNull; +import java.math.BigDecimal; +import java.util.List; +import java.util.UUID; import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.models.error.SimpleErrorInfo; diff --git a/backend/src/main/java/com/bakdata/conquery/io/external/form/FormBackendVersion.java b/backend/src/main/java/com/bakdata/conquery/io/external/form/FormBackendVersion.java index c661159effa..ad0b4b5fd53 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/external/form/FormBackendVersion.java +++ b/backend/src/main/java/com/bakdata/conquery/io/external/form/FormBackendVersion.java @@ -4,7 +4,7 @@ public record FormBackendVersion( - String version, - ZonedDateTime buildTime + String version, + ZonedDateTime buildTime ) { } diff --git a/backend/src/main/java/com/bakdata/conquery/io/freemarker/Freemarker.java b/backend/src/main/java/com/bakdata/conquery/io/freemarker/Freemarker.java index 66faafc9050..93b3c7df0f3 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/freemarker/Freemarker.java +++ b/backend/src/main/java/com/bakdata/conquery/io/freemarker/Freemarker.java @@ -14,9 +14,9 @@ @Slf4j public class Freemarker { - + public static final Version VERSION = Configuration.VERSION_2_3_27; - + public static final FreemarkerViewRenderer HTML_RENDERER = rendererForHtml(); @@ -25,15 +25,20 @@ private static FreemarkerViewRenderer rendererForHtml() { freemarker.configure(Freemarker.asMap()); return freemarker; } - + private static Map asMap() { - return ImmutableMap - .builder() - .put(DEFAULT_ENCODING_KEY_CAMEL_CASE, StandardCharsets.UTF_8.name()) + return ImmutableMap.builder() + .put( + DEFAULT_ENCODING_KEY_CAMEL_CASE, + StandardCharsets.UTF_8.name()) .put(LOCALIZED_LOOKUP_KEY_CAMEL_CASE, Boolean.TRUE.toString()) - .put(WHITESPACE_STRIPPING_KEY_CAMEL_CASE, Boolean.TRUE.toString()) + .put( + WHITESPACE_STRIPPING_KEY_CAMEL_CASE, + Boolean.TRUE.toString()) .put(WRAP_UNCHECKED_EXCEPTIONS_KEY_CAMEL_CASE, Boolean.TRUE.toString()) - .put(LOG_TEMPLATE_EXCEPTIONS_KEY_CAMEL_CASE, Boolean.FALSE.toString()) + .put( + LOG_TEMPLATE_EXCEPTIONS_KEY_CAMEL_CASE, + Boolean.FALSE.toString()) .put(NUMBER_FORMAT_KEY_CAMEL_CASE, "computer") .put(LOCALE_KEY, "") .build(); diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/ConquerySerializersModule.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/ConquerySerializersModule.java index d48df913956..09e9956df15 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/ConquerySerializersModule.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/ConquerySerializersModule.java @@ -43,7 +43,7 @@ private ConquerySerializersModule() { public boolean canCreateUsingDefault() { return true; } - + @Override public Object createUsingDefault(DeserializationContext ctxt) throws IOException { return HashBiMap.create(); @@ -52,9 +52,7 @@ public Object createUsingDefault(DeserializationContext ctxt) throws IOException addDeserializer(ClassToInstanceMap.class, new ClassToInstanceMapDeserializer()); //register IdKeySerializer for all id types - List> idTypes = - CPSTypeIdResolver.SCAN_RESULT.getSubclasses(Id.class.getName()) - .loadClasses(); + List> idTypes = CPSTypeIdResolver.SCAN_RESULT.getSubclasses(Id.class.getName()).loadClasses(); for (Class type : idTypes) { addKeyDeserializer(type, new IdKeyDeserializer<>()); @@ -67,4 +65,4 @@ public Object createUsingDefault(DeserializationContext ctxt) throws IOException addDeserializer(Locale.class, new LocaleDeserializer()); addSerializer(Locale.class, new LocaleSerializer()); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/Initializing.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/Initializing.java index ff01332b7ac..f0b879d64ad 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/Initializing.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/Initializing.java @@ -11,7 +11,6 @@ * @JsonDeserialize(converter = Initializing.Converter.class ) * } * - * @param * @implNote Every class that inherits from an initializing class needs to define its own Converter. * Otherwise, the class is parsed as the super class, and its overrides are not called. */ @@ -26,8 +25,7 @@ public T convert(T value) { try { value.init(); return value; - } - catch (Exception e) { + } catch (Exception e) { Throwables.throwIfUnchecked(e); throw new IllegalStateException("Could not initialize %s".formatted(value), e); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/Injectable.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/Injectable.java index 5e46b3a2dd9..39833dff47d 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/Injectable.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/Injectable.java @@ -9,7 +9,7 @@ public interface Injectable { /** - * See {@link Injectable#injectIntoNew(ObjectMapper)} + * See {@link Injectable#injectIntoNew(ObjectMapper)} */ default ObjectReader injectIntoNew(ObjectReader reader) { // If is already MutableInjectable, add my values to other, else begin from scratch. @@ -21,7 +21,7 @@ default ObjectReader injectIntoNew(ObjectReader reader) { } /** - * Creates a copy of the provided mapper and its injected values and adds the caller to the new copy + * Creates a copy of the provided mapper and its injected values and adds the caller to the new copy * @param mapper the blueprint mapper to use which remains untouched * @return a new mapper with this injected */ @@ -30,11 +30,11 @@ default ObjectMapper injectIntoNew(ObjectMapper mapper) { if (mapper.getInjectableValues() instanceof MutableInjectableValues) { return mapper.copy() - .setInjectableValues(inject(((MutableInjectableValues) mapper.getInjectableValues()).copy())); + .setInjectableValues( + inject(((MutableInjectableValues) mapper.getInjectableValues()).copy())); } // TODO unsure if overriding is expected here from the user - return mapper.copy() - .setInjectableValues(inject(new MutableInjectableValues())); + return mapper.copy().setInjectableValues(inject(new MutableInjectableValues())); } /** @@ -47,7 +47,8 @@ default ObjectMapper injectInto(ObjectMapper mapper) { mapper.setInjectableValues(inject(((MutableInjectableValues) mapper.getInjectableValues()))); return mapper; } - throw new IllegalStateException("Cannot add additional injectables if the mapper does not provide MutableInjectableValues"); + throw new IllegalStateException( + "Cannot add additional injectables if the mapper does not provide MutableInjectableValues"); } MutableInjectableValues inject(MutableInjectableValues values); diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/Jackson.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/Jackson.java index e54b5c3c3bd..527f0b9d431 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/Jackson.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/Jackson.java @@ -48,34 +48,59 @@ public static ObjectMapper copyMapperAndInjectables(ObjectMapper om) { public static T configure(T objectMapper) { objectMapper.enable(MapperFeature.PROPAGATE_TRANSIENT_MARKER) - .enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY) - .enable(Feature.ALLOW_UNQUOTED_FIELD_NAMES) - .enable(Feature.ALLOW_COMMENTS) - .enable(Feature.ALLOW_UNQUOTED_CONTROL_CHARS) - .enable(DeserializationFeature.FAIL_ON_INVALID_SUBTYPE) - .enable(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES) - .enable(DeserializationFeature.FAIL_ON_NUMBERS_FOR_ENUMS) - .enable(DeserializationFeature.FAIL_ON_READING_DUP_TREE_KEY) - .enable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) - .enable(DeserializationFeature.FAIL_ON_UNRESOLVED_OBJECT_IDS) - .enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS) - .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) - .setLocale(Locale.ROOT) - .disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET) - .enable(SerializationFeature.WRITE_EMPTY_JSON_ARRAYS) - .enable(SerializationFeature.WRITE_NULL_MAP_VALUES) - .registerModule(new JavaTimeModule()) - .registerModule(new ParameterNamesModule()) - .registerModule(new GuavaModule()) - .registerModule(new BlackbirdModule()) - .registerModule(ConquerySerializersModule.INSTANCE) - .setSerializationInclusion(Include.ALWAYS) - .setDefaultPropertyInclusion(Include.ALWAYS) - //.setAnnotationIntrospector(new RestrictingAnnotationIntrospector()) - .setInjectableValues(new MutableInjectableValues()) - .addMixIn(Permission.class, ConqueryPermission.class) - .addMixIn(Object2IntMap.class, Object2IntMapMixIn.class) - .addMixIn(DefaultSocketSessionConfig.class, DefaultSocketSessionConfigMixIn.class); + .enable( + DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY) + .enable(Feature.ALLOW_UNQUOTED_FIELD_NAMES) + .enable( + Feature.ALLOW_COMMENTS) + .enable(Feature.ALLOW_UNQUOTED_CONTROL_CHARS) + .enable( + DeserializationFeature.FAIL_ON_INVALID_SUBTYPE) + .enable( + DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES) + .enable( + DeserializationFeature.FAIL_ON_NUMBERS_FOR_ENUMS) + .enable( + DeserializationFeature.FAIL_ON_READING_DUP_TREE_KEY) + .enable( + DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) + .enable( + DeserializationFeature.FAIL_ON_UNRESOLVED_OBJECT_IDS) + .enable( + DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS) + .disable( + SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) + .setLocale( + Locale.ROOT) + .disable( + JsonGenerator.Feature.AUTO_CLOSE_TARGET) + .enable( + SerializationFeature.WRITE_EMPTY_JSON_ARRAYS) + .enable( + SerializationFeature.WRITE_NULL_MAP_VALUES) + .registerModule( + new JavaTimeModule()) + .registerModule( + new ParameterNamesModule()) + .registerModule( + new GuavaModule()) + .registerModule( + new BlackbirdModule()) + .registerModule( + ConquerySerializersModule.INSTANCE) + .setSerializationInclusion( + Include.ALWAYS) + .setDefaultPropertyInclusion( + Include.ALWAYS) + //.setAnnotationIntrospector(new RestrictingAnnotationIntrospector()) + .setInjectableValues(new MutableInjectableValues()) + .addMixIn( + Permission.class, + ConqueryPermission.class) + .addMixIn(Object2IntMap.class, Object2IntMapMixIn.class) + .addMixIn( + DefaultSocketSessionConfig.class, + DefaultSocketSessionConfigMixIn.class); return objectMapper; } diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/JacksonUtil.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/JacksonUtil.java index 0e19add7fb2..9dec09d4c3d 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/JacksonUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/JacksonUtil.java @@ -12,7 +12,8 @@ import org.apache.commons.text.translate.JavaUnicodeEscaper; import org.apache.mina.core.buffer.IoBuffer; -@Slf4j @UtilityClass +@Slf4j +@UtilityClass public class JacksonUtil { /** @@ -55,9 +56,7 @@ public static String toJsonDebug(InputStream is) { case VALUE_STRING: String value = StringUtils.abbreviate(parser.getText(), 50); - value = JavaUnicodeEscaper - .outsideOf(32, 0x7e) - .translate(value); + value = JavaUnicodeEscaper.outsideOf(32, 0x7e).translate(value); sb.append('"').append(value).append("\","); break; default: @@ -67,19 +66,16 @@ public static String toJsonDebug(InputStream is) { } } return sb.toString(); - } - catch (Exception e) { + } catch (Exception e) { log.warn("Failed to create the debug json", e); if (!sb.isEmpty()) { sb.append("DEBUG_JSON_ERROR"); - } - else { + } else { try { is.reset(); sb.append(logHex(is)); - } - catch (Exception hexException) { + } catch (Exception hexException) { log.error("Unable to generate hex dump of input stream", e); sb.append("HEX_DUMP_ERROR"); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/MutableInjectableValues.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/MutableInjectableValues.java index d5c36dd662e..58b4a6462d6 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/MutableInjectableValues.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/MutableInjectableValues.java @@ -22,18 +22,22 @@ public MutableInjectableValues add(Class type, T value) { } @Override - public Object findInjectableValue(Object valueId, DeserializationContext ctxt, BeanProperty forProperty, Object beanInstance) throws JsonMappingException { + public Object findInjectableValue( + Object valueId, + DeserializationContext ctxt, + BeanProperty forProperty, + Object beanInstance) throws JsonMappingException { if (valueId instanceof Class clazz) { return findInjectableValue(clazz.getName(), ctxt, forProperty, beanInstance); - } - else if (valueId instanceof String key) { + } else if (valueId instanceof String key) { return values.get(key); } - ctxt.reportBadDefinition(ClassUtil.classOf(valueId), - String.format( - "Unrecognized inject value id type (%s), expecting String or Class", - ClassUtil.classNameOf(valueId) - ) + ctxt.reportBadDefinition( + ClassUtil.classOf(valueId), + String.format( + "Unrecognized inject value id type (%s), expecting String or Class", + ClassUtil.classNameOf(valueId) + ) ); return null; // Not reached diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/View.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/View.java index a514c9a9eb2..3fb8ef8191f 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/View.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/View.java @@ -24,7 +24,8 @@ public interface View { * @see View.ApiManagerPersistence for classes that is propagated form the api unto the shards, * but have fields that should not be sent to the shards. */ - interface Api extends View {} + interface Api extends View { + } /** * View class for fields that should be written to a storage. @@ -32,9 +33,11 @@ interface Api extends View {} * @see View.Internal for fields that are shared between manager and shards and should be persisted. */ interface Persistence extends View { - interface Manager extends Persistence {} + interface Manager extends Persistence { + } - interface Shard extends Persistence {} + interface Shard extends Persistence { + } } /** @@ -42,7 +45,8 @@ interface Shard extends Persistence {} * * @see View.Internal for fields that are shared between manager and shards and should be persisted. */ - interface InternalCommunication extends View {} + interface InternalCommunication extends View { + } /** * Meta annotation for fields that are internally created, used, persisted and shared between manager node and shard nodes, @@ -51,7 +55,8 @@ interface InternalCommunication extends View {} @Retention(RetentionPolicy.RUNTIME) @JacksonAnnotationsInside @JsonView({View.InternalCommunication.class, View.Persistence.class}) - @interface Internal {} + @interface Internal { + } /** * Meta annotation for fields that are written or read through the REST api and stored on the manager, but never touch a shard node. @@ -59,5 +64,6 @@ interface InternalCommunication extends View {} @Retention(RetentionPolicy.RUNTIME) @JacksonAnnotationsInside @JsonView({View.Api.class, View.Persistence.Manager.class}) - @interface ApiManagerPersistence {} + @interface ApiManagerPersistence { + } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/mixin/DefaultSocketSessionConfigMixIn.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/mixin/DefaultSocketSessionConfigMixIn.java index 0c86bf275dd..cf442c0f13e 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/mixin/DefaultSocketSessionConfigMixIn.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/mixin/DefaultSocketSessionConfigMixIn.java @@ -6,12 +6,8 @@ /** * MixIn to suppress artificial properties of {@link DefaultSocketSessionConfig}. */ -@JsonIgnoreProperties(value = { - "throughputCalculationIntervalInMillis", - "readerIdleTimeInMillis", - "writeTimeoutInMillis", - "writerIdleTimeInMillis", - "bothIdleTimeInMillis" -}) +@JsonIgnoreProperties( + value = {"throughputCalculationIntervalInMillis", "readerIdleTimeInMillis", "writeTimeoutInMillis", "writerIdleTimeInMillis", "bothIdleTimeInMillis" + }) public class DefaultSocketSessionConfigMixIn extends DefaultSocketSessionConfig { } diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/mixin/Object2IntMapMixIn.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/mixin/Object2IntMapMixIn.java index eac1ca83464..035c7857b0d 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/mixin/Object2IntMapMixIn.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/mixin/Object2IntMapMixIn.java @@ -44,30 +44,41 @@ public Deserializer(MapDeserializer mapDeserializer) { } @Override - public Object2IntMap deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JacksonException { + public Object2IntMap deserialize( + JsonParser p, + DeserializationContext ctxt) throws IOException, JacksonException { final Object2IntOpenHashMap map = new Object2IntOpenHashMap<>(); Object2IntOpenHashMap map1 = map; return (Object2IntMap) mapDeserializer.deserialize(p, ctxt, (Map) map1); } @Override - public JsonDeserializer createContextual(DeserializationContext ctxt, BeanProperty property) throws JsonMappingException { + public JsonDeserializer createContextual( + DeserializationContext ctxt, + BeanProperty property) throws JsonMappingException { final JavaType mapType = ctxt.constructType(Object2IntMap.class); KeyDeserializer keyDeserializer; final JavaType keyType = ctxt.getContextualType().getKeyType(); try { keyDeserializer = ctxt.findKeyDeserializer(keyType, property); - } - catch (InvalidDefinitionException e) { + } catch (InvalidDefinitionException e) { log.trace("Falling back to delegating key deserializer for type: {} ", keyType); - final JsonDeserializer contextualKeyDeserializer = ctxt.findContextualValueDeserializer(keyType, property); - keyDeserializer = StdKeyDeserializers.constructDelegatingKeyDeserializer(ctxt.getConfig(), keyType, contextualKeyDeserializer); + final JsonDeserializer contextualKeyDeserializer = ctxt.findContextualValueDeserializer( + keyType, + property); + keyDeserializer = StdKeyDeserializers.constructDelegatingKeyDeserializer( + ctxt.getConfig(), + keyType, + contextualKeyDeserializer); } final JavaType valueType = ctxt.getContextualType().getContentType(); - final JsonDeserializer valueDeserializer = ctxt.findContextualValueDeserializer(valueType, property); + final JsonDeserializer valueDeserializer = ctxt.findContextualValueDeserializer( + valueType, + property); final StdValueInstantiator valueInstantiator = new StdValueInstantiator(ctxt.getConfig(), valueType); - return new Deserializer<>(new MapDeserializer(mapType, valueInstantiator, keyDeserializer, valueDeserializer, null)); + return new Deserializer<>( + new MapDeserializer(mapType, valueInstantiator, keyDeserializer, valueDeserializer, null)); } } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/BitSetDeserializer.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/BitSetDeserializer.java index b71609dc96a..ba7ace2742a 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/BitSetDeserializer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/BitSetDeserializer.java @@ -5,7 +5,6 @@ import java.util.BitSet; import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.deser.std.StdDeserializer; @@ -15,8 +14,8 @@ protected BitSetDeserializer() { } @Override - public BitSet deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException { - try(final ByteArrayOutputStream buffer = new ByteArrayOutputStream()) { + public BitSet deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { + try (final ByteArrayOutputStream buffer = new ByteArrayOutputStream()) { p.readBinaryValue(buffer); final byte[] bytes = buffer.toByteArray(); diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CBlockDeserializer.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CBlockDeserializer.java index 08ebc2de1a2..c5d5aea866a 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CBlockDeserializer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CBlockDeserializer.java @@ -29,10 +29,12 @@ public class CBlockDeserializer extends JsonDeserializer implements Cont @SuppressWarnings({"rawtypes", "unchecked"}) @Override - public JsonDeserializer createContextual(DeserializationContext ctxt, BeanProperty property) throws JsonMappingException { - JavaType type = Optional - .ofNullable(ctxt.getContextualType()) - .orElseGet(Optional.ofNullable(property).map(BeanProperty::getType)::get); + public JsonDeserializer createContextual( + DeserializationContext ctxt, + BeanProperty property) throws JsonMappingException { + JavaType type = Optional.ofNullable(ctxt.getContextualType()) + .orElseGet( + Optional.ofNullable(property).map(BeanProperty::getType)::get); while (type.isContainerType()) { type = type.getContentType(); @@ -46,7 +48,10 @@ public JsonDeserializer createContextual(DeserializationContext ctxt, BeanPro } @Override - public CBlock deserializeWithType(JsonParser p, DeserializationContext ctxt, TypeDeserializer typeDeserializer) throws IOException { + public CBlock deserializeWithType( + JsonParser p, + DeserializationContext ctxt, + TypeDeserializer typeDeserializer) throws IOException { return this.deserialize(p, ctxt); } @@ -68,7 +73,8 @@ public CBlock deserialize(JsonParser p, DeserializationContext ctxt) throws IOEx } log.trace("Getting Elements for local ids: {}", mostSpecificChildren); - block.getMostSpecificChildren()[event] = concept.getElementByLocalIdPath(mostSpecificChildren).getPrefix(); + block.getMostSpecificChildren()[event] = concept.getElementByLocalIdPath( + mostSpecificChildren).getPrefix(); } } return block; diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CDateRangeDeserializer.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CDateRangeDeserializer.java index 43b803a1a4a..568e2308cd5 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CDateRangeDeserializer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CDateRangeDeserializer.java @@ -12,16 +12,18 @@ public class CDateRangeDeserializer extends StdDeserializer { - private final DateReader dateReader; + private final DateReader dateReader; - protected CDateRangeDeserializer(DateReader dateReader) { - super(CDateRange.class); - this.dateReader = dateReader; - } + protected CDateRangeDeserializer(DateReader dateReader) { + super(CDateRange.class); + this.dateReader = dateReader; + } - @SneakyThrows - @Override - public CDateRange deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException { - return dateReader.parseToCDateRange(p.getText()); - } + @SneakyThrows + @Override + public CDateRange deserialize( + JsonParser p, + DeserializationContext ctxt) throws IOException, JsonProcessingException { + return dateReader.parseToCDateRange(p.getText()); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CDateSetDeserializer.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CDateSetDeserializer.java index 63ee1280bf3..da695b17d0f 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CDateSetDeserializer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CDateSetDeserializer.java @@ -13,33 +13,33 @@ public class CDateSetDeserializer extends StdDeserializer { - private static final long serialVersionUID = 1L; + private static final long serialVersionUID = 1L; - private final DateReader formats; + private final DateReader formats; - public CDateSetDeserializer(DateReader formats) { - super(CDateSet.class); - this.formats = formats; - } + public CDateSetDeserializer(DateReader formats) { + super(CDateSet.class); + this.formats = formats; + } - @Override - public CDateSet deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException { - if (p.currentToken() == JsonToken.START_ARRAY) { - int[][] ints = p.readValueAs(int[][].class); + @Override + public CDateSet deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException { + if (p.currentToken() == JsonToken.START_ARRAY) { + int[][] ints = p.readValueAs(int[][].class); - CDateSet set = CDateSet.createEmpty(); - for (int i = 0; i < ints.length; i++) { - set.add(CDateRange.of(ints[i][0], ints[i][1])); - } - return set; - } + CDateSet set = CDateSet.createEmpty(); + for (int i = 0; i < ints.length; i++) { + set.add(CDateRange.of(ints[i][0], ints[i][1])); + } + return set; + } - if (p.currentToken() == JsonToken.VALUE_STRING) { - return formats.parseToCDateSet(p.readValueAs(String.class)); - } + if (p.currentToken() == JsonToken.VALUE_STRING) { + return formats.parseToCDateSet(p.readValueAs(String.class)); + } - return (CDateSet) ctxt.handleUnexpectedToken(CDateSet.class, p.currentToken(), p, "can't deserialize CDateSet"); + return (CDateSet) ctxt.handleUnexpectedToken(CDateSet.class, p.currentToken(), p, "can't deserialize CDateSet"); - } + } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CDateSetSerializer.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CDateSetSerializer.java index 62187b39a55..47f802a5614 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CDateSetSerializer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CDateSetSerializer.java @@ -19,7 +19,7 @@ public CDateSetSerializer() { @Override public void serialize(CDateSet value, JsonGenerator gen, SerializerProvider provider) throws IOException { gen.writeStartArray(value.asRanges().size()); - for(CDateRange range : value.asRanges()) { + for (CDateRange range : value.asRanges()) { gen.writeStartArray(2); gen.writeNumber(range.getMinValue()); gen.writeNumber(range.getMaxValue()); @@ -27,4 +27,4 @@ public void serialize(CDateSet value, JsonGenerator gen, SerializerProvider prov } gen.writeEndArray(); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CharArrayBufferDeserializer.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CharArrayBufferDeserializer.java index 59cddf77ee1..c38944c0713 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CharArrayBufferDeserializer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CharArrayBufferDeserializer.java @@ -1,5 +1,8 @@ package com.bakdata.conquery.io.jackson.serializer; +import java.io.IOException; +import java.util.Arrays; + import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonToken; @@ -7,19 +10,18 @@ import com.fasterxml.jackson.databind.JsonDeserializer; import org.apache.http.util.CharArrayBuffer; -import java.io.IOException; -import java.util.Arrays; - /** * This custom serializer treats an incoming text node as a mutable string, which is important for sensitve data. * Doing so, allows us to clear buffers. Since {@link String} is immutable it is inappropriate. */ public class CharArrayBufferDeserializer extends JsonDeserializer { @Override - public CharArrayBuffer deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException { + public CharArrayBuffer deserialize( + JsonParser p, + DeserializationContext ctxt) throws IOException, JsonProcessingException { final JsonToken currentToken = p.getCurrentToken(); - if(currentToken != JsonToken.VALUE_STRING){ - ctxt.handleUnexpectedToken(CharArrayBuffer.class, currentToken, p,"cannot deserialize CharArrayBuffer"); + if (currentToken != JsonToken.VALUE_STRING) { + ctxt.handleUnexpectedToken(CharArrayBuffer.class, currentToken, p, "cannot deserialize CharArrayBuffer"); } final char[] chars = p.readValueAs(char[].class); final CharArrayBuffer charArrayBuffer = new CharArrayBuffer(chars.length); diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CharArrayBufferSerializer.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CharArrayBufferSerializer.java index 860229a8ffc..64d54a4ade9 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CharArrayBufferSerializer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CharArrayBufferSerializer.java @@ -1,13 +1,13 @@ package com.bakdata.conquery.io.jackson.serializer; +import java.io.IOException; +import java.util.Arrays; + import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.SerializerProvider; import org.apache.http.util.CharArrayBuffer; -import java.io.IOException; -import java.util.Arrays; - /** * This serializer avoids that the character sequence is transformed into an immutable String, with which we would lose control over invalidation of its content. */ diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/ClassToInstanceMapDeserializer.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/ClassToInstanceMapDeserializer.java index 2c61d02fa1c..1bbc018a1d2 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/ClassToInstanceMapDeserializer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/ClassToInstanceMapDeserializer.java @@ -18,18 +18,20 @@ public ClassToInstanceMapDeserializer() { } @Override - public ClassToInstanceMap deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException { + public ClassToInstanceMap deserialize( + JsonParser p, + DeserializationContext ctxt) throws IOException, JsonProcessingException { ClassToInstanceMap map = MutableClassToInstanceMap.create(); - if(p.currentToken()==JsonToken.START_OBJECT) { + if (p.currentToken() == JsonToken.START_OBJECT) { p.nextToken(); } - while(true) { - if(p.currentToken()==JsonToken.END_OBJECT) { + while (true) { + if (p.currentToken() == JsonToken.END_OBJECT) { return map; } String typeName = p.getCurrentName(); - if(typeName == null) { - return (ClassToInstanceMap)ctxt.handleUnexpectedToken(ClassToInstanceMap.class, p); + if (typeName == null) { + return (ClassToInstanceMap) ctxt.handleUnexpectedToken(ClassToInstanceMap.class, p); } try { Class type = Class.forName(typeName); @@ -37,9 +39,8 @@ public ClassToInstanceMap deserialize(JsonParser p, DeserializationContext ct Object value = ctxt.readValue(p, type); p.nextToken(); map.putInstance(type, value); - } - catch (ClassNotFoundException e) { - throw new RuntimeException("Could not resolve class "+typeName, e); + } catch (ClassNotFoundException e) { + throw new RuntimeException("Could not resolve class " + typeName, e); } } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/ConqueryDoubleSerializer.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/ConqueryDoubleSerializer.java index 9061efa5ec7..5986fc9221b 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/ConqueryDoubleSerializer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/ConqueryDoubleSerializer.java @@ -12,13 +12,12 @@ public class ConqueryDoubleSerializer extends JsonSerializer { public Class handledType() { return Double.class; } - + @Override public void serialize(Double value, JsonGenerator gen, SerializerProvider serializers) throws IOException { - if(value.isInfinite() || value.isNaN()) { + if (value.isInfinite() || value.isNaN()) { gen.writeNull(); - } - else { + } else { gen.writeNumber(value.doubleValue()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CurrencyUnitDeserializer.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CurrencyUnitDeserializer.java index 66bbf35fc08..df0d3a0a2a3 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CurrencyUnitDeserializer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CurrencyUnitDeserializer.java @@ -15,7 +15,7 @@ public class CurrencyUnitDeserializer extends StdScalarDeserializer { public CurrencyUnitDeserializer() { super(String.class); } - + @Override public Currency deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { if (p.currentTokenId() != JsonTokenId.ID_STRING) { @@ -26,6 +26,5 @@ public Currency deserialize(JsonParser p, DeserializationContext ctxt) throws IO return Currency.getInstance(text); } - } diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/DatasetParamInjector.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/DatasetParamInjector.java index 9acf18a660b..eaae743d5d7 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/DatasetParamInjector.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/DatasetParamInjector.java @@ -1,10 +1,10 @@ package com.bakdata.conquery.io.jackson.serializer; -import java.io.IOException; import jakarta.inject.Inject; import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.container.ContainerRequestFilter; import jakarta.ws.rs.core.MultivaluedMap; +import java.io.IOException; import com.bakdata.conquery.io.jackson.Injectable; import com.bakdata.conquery.io.jackson.MutableInjectableValues; @@ -42,15 +42,20 @@ public static class Modifier extends ObjectReaderModifier implements Injectable public final DatasetRegistry registry; @Override - public ObjectReader modify(EndpointConfigBase endpoint, MultivaluedMap httpHeaders, JavaType resultType, ObjectReader reader, JsonParser p) - throws IOException { + public ObjectReader modify( + EndpointConfigBase endpoint, + MultivaluedMap httpHeaders, + JavaType resultType, + ObjectReader reader, + JsonParser p) throws IOException { return injectIntoNew(reader); } @Override public MutableInjectableValues inject(MutableInjectableValues values) { if (pathParams.containsKey(ResourceConstants.DATASET)) { - final DatasetId datasetId = DatasetId.Parser.INSTANCE.parse(pathParams.getFirst(ResourceConstants.DATASET)); + final DatasetId datasetId = DatasetId.Parser.INSTANCE.parse( + pathParams.getFirst(ResourceConstants.DATASET)); datasetId.setDomain(registry); // this is just interning @@ -60,4 +65,4 @@ public MutableInjectableValues inject(MutableInjectableValues values) { } } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/FormatedDateDeserializer.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/FormatedDateDeserializer.java index ead4a01e792..9f58065a88b 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/FormatedDateDeserializer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/FormatedDateDeserializer.java @@ -1,26 +1,26 @@ package com.bakdata.conquery.io.jackson.serializer; +import java.io.IOException; +import java.time.LocalDate; + import com.bakdata.conquery.util.DateReader; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.deser.std.StdDeserializer; import lombok.SneakyThrows; -import java.io.IOException; -import java.time.LocalDate; - public class FormatedDateDeserializer extends StdDeserializer { - private final DateReader formats; + private final DateReader formats; - public FormatedDateDeserializer(DateReader formats) { - super(LocalDate.class); - this.formats = formats; - } + public FormatedDateDeserializer(DateReader formats) { + super(LocalDate.class); + this.formats = formats; + } - @SneakyThrows - @Override - public LocalDate deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { - return formats.parseToLocalDate(p.getText()); - } + @SneakyThrows + @Override + public LocalDate deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { + return formats.parseToLocalDate(p.getText()); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/IdDeserializer.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/IdDeserializer.java index b37d90cf8fc..ea25749096a 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/IdDeserializer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/IdDeserializer.java @@ -41,7 +41,8 @@ public class IdDeserializer> extends JsonDeserializer im @Override public JsonDeserializer createContextual(DeserializationContext ctxt, BeanProperty property) { JavaType type = Optional.ofNullable(ctxt.getContextualType()) - .orElseGet(Optional.ofNullable(property).map(BeanProperty::getType)::get); + .orElseGet( + Optional.ofNullable(property).map(BeanProperty::getType)::get); while (type.isContainerType()) { type = type.getContentType(); @@ -50,15 +51,18 @@ public JsonDeserializer createContextual(DeserializationContext ctxt, BeanPro IdUtil.Parser, ?>> parser = IdUtil.createParser((Class) idClass); return new IdDeserializer( - idClass, - parser, - //we only need to check for the dataset prefix if the id requires it - NamespacedId.class.isAssignableFrom(idClass) + idClass, + parser, + //we only need to check for the dataset prefix if the id requires it + NamespacedId.class.isAssignableFrom(idClass) ); } @Override - public ID deserializeWithType(JsonParser p, DeserializationContext ctxt, TypeDeserializer typeDeserializer) throws IOException { + public ID deserializeWithType( + JsonParser p, + DeserializationContext ctxt, + TypeDeserializer typeDeserializer) throws IOException { return this.deserialize(p, ctxt); } @@ -68,7 +72,11 @@ public ID deserialize(JsonParser parser, DeserializationContext ctxt) throws IOE JsonToken currentToken = parser.getCurrentToken(); if (currentToken != JsonToken.VALUE_STRING) { - return (ID) ctxt.handleUnexpectedToken(Id.class, currentToken, parser, "name references should be strings. Was: " + currentToken); + return (ID) ctxt.handleUnexpectedToken( + Id.class, + currentToken, + parser, + "name references should be strings. Was: " + currentToken); } String text = parser.getText(); @@ -85,14 +93,19 @@ public ID deserialize(JsonParser parser, DeserializationContext ctxt) throws IOE } return id; - } - catch (Exception e) { - return (ID) ctxt.handleWeirdStringValue(idClass, text, "Could not parse `" + idClass.getSimpleName() + "` from `" + text + "`: " + e.getMessage()); + } catch (Exception e) { + return (ID) ctxt.handleWeirdStringValue( + idClass, + text, + "Could not parse `" + idClass.getSimpleName() + "` from `" + text + "`: " + e.getMessage()); } } - public static > ID deserializeId(String text, IdUtil.Parser idParser, boolean checkForInjectedPrefix, DeserializationContext ctx) - throws JsonMappingException { + public static > ID deserializeId( + String text, + IdUtil.Parser idParser, + boolean checkForInjectedPrefix, + DeserializationContext ctx) throws JsonMappingException { String prefix = null; @@ -100,9 +113,9 @@ public ID deserialize(JsonParser parser, DeserializationContext ctxt) throws IOE prefix = findDatasetName(ctx); } - List components = prefix != null ? - IdUtil.Parser.asComponents(prefix, text) : - IdUtil.Parser.asComponents(text); + List components = prefix != null ? IdUtil.Parser.asComponents( + prefix, + text) : IdUtil.Parser.asComponents(text); IdInterner interner = IdInterner.get(ctx); @@ -118,7 +131,10 @@ public ID deserialize(JsonParser parser, DeserializationContext ctxt) throws IOE return id; } - public static void setResolver(Id id, MetaStorage metaStorage, NamespacedStorageProvider namespacedStorageProvider) { + public static void setResolver( + Id id, + MetaStorage metaStorage, + NamespacedStorageProvider namespacedStorageProvider) { // Set resolvers in this id and subIds final Set> ids = new HashSet<>(); id.collectIds(ids); diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/IdKeyDeserializer.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/IdKeyDeserializer.java index 8f7c03db508..a6aa9e4bb00 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/IdKeyDeserializer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/IdKeyDeserializer.java @@ -26,8 +26,7 @@ public class IdKeyDeserializer> extends KeyDeserializer impl public ID deserializeKey(String key, DeserializationContext ctxt) { try { return IdDeserializer.deserializeId(key, idParser, parsePrefixed, ctxt); - } - catch (Exception e) { + } catch (Exception e) { throw new IllegalArgumentException("Could not parse an " + idClass.getSimpleName() + " from " + key, e); } } @@ -35,7 +34,8 @@ public ID deserializeKey(String key, DeserializationContext ctxt) { @Override public KeyDeserializer createContextual(DeserializationContext ctxt, BeanProperty property) { JavaType type = Optional.ofNullable(ctxt.getContextualType()) - .orElseGet(Optional.ofNullable(property).map(BeanProperty::getType)::get); + .orElseGet( + Optional.ofNullable(property).map(BeanProperty::getType)::get); //since this a key serializer property should point to a map type = type.getKeyType(); diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/Int2IntArrayMapDeserializer.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/Int2IntArrayMapDeserializer.java index 4dec1967d17..852ff271863 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/Int2IntArrayMapDeserializer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/Int2IntArrayMapDeserializer.java @@ -3,7 +3,6 @@ import java.io.IOException; import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.deser.std.StdDeserializer; import it.unimi.dsi.fastutil.ints.Int2IntArrayMap; @@ -16,12 +15,15 @@ protected Int2IntArrayMapDeserializer() { } @Override - public Int2IntArrayMap deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException { - final Int2IntMapSerializer.SerializationContainer container = p.readValueAs(Int2IntMapSerializer.SerializationContainer.class); + public Int2IntArrayMap deserialize( + JsonParser p, + DeserializationContext ctxt) throws IOException { + final Int2IntMapSerializer.SerializationContainer container = p.readValueAs( + Int2IntMapSerializer.SerializationContainer.class); final Int2IntArrayMap out = new Int2IntArrayMap( - container.getKeys().stream().mapToInt(Integer::intValue).toArray(), - container.getValues().stream().mapToInt(Integer::intValue).toArray() + container.getKeys().stream().mapToInt(Integer::intValue).toArray(), + container.getValues().stream().mapToInt(Integer::intValue).toArray() ); out.defaultReturnValue(container.getDefaultValue()); diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/LocaleDeserializer.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/LocaleDeserializer.java index a5f101ec79c..cac6b54f04f 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/LocaleDeserializer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/LocaleDeserializer.java @@ -31,7 +31,10 @@ public Locale deserialize(JsonParser p, DeserializationContext ctxt) throws IOEx if (StringUtils.isNotBlank(languageTag) && !languageTag.equals(ROOT_LOCALE_TAG) && locale == Locale.ROOT) { // When Locale#forLanguageTag does not recognize the tag it defaults to Locale.ROOT // This should only be intended if the tag was blank or "und" (undefined ~= Locale.ROOT.toLanguageTag()) - throw MismatchedInputException.from(p, Locale.class, "Unable to map language tag '" + languageTag + "' to locale"); + throw MismatchedInputException.from( + p, + Locale.class, + "Unable to map language tag '" + languageTag + "' to locale"); } return locale; diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/MediaTypeDeserializer.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/MediaTypeDeserializer.java index 46f5b338cc5..1ffe2a2b758 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/MediaTypeDeserializer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/MediaTypeDeserializer.java @@ -1,16 +1,17 @@ package com.bakdata.conquery.io.jackson.serializer; +import jakarta.ws.rs.core.MediaType; +import java.io.IOException; + import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonDeserializer; -import jakarta.ws.rs.core.MediaType; -import java.io.IOException; - public class MediaTypeDeserializer extends JsonDeserializer { @Override - public MediaType deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { + public MediaType deserialize( + JsonParser jsonParser, + DeserializationContext deserializationContext) throws IOException { return MediaType.valueOf(jsonParser.getText()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/jersey/ExtraMimeTypes.java b/backend/src/main/java/com/bakdata/conquery/io/jersey/ExtraMimeTypes.java index 62702388fda..f64ba9d08d9 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jersey/ExtraMimeTypes.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jersey/ExtraMimeTypes.java @@ -4,6 +4,6 @@ public class ExtraMimeTypes { public static final String SMILE_STRING = "application/x-jackson-smile"; public static final String JSON_STRING = "application/json"; - - + + } diff --git a/backend/src/main/java/com/bakdata/conquery/io/jersey/IdPathParamConverterProvider.java b/backend/src/main/java/com/bakdata/conquery/io/jersey/IdPathParamConverterProvider.java index 25170bbf856..259150dc78e 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jersey/IdPathParamConverterProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jersey/IdPathParamConverterProvider.java @@ -1,11 +1,11 @@ package com.bakdata.conquery.io.jersey; -import java.lang.annotation.Annotation; -import java.lang.reflect.Type; -import java.util.Objects; import jakarta.inject.Inject; import jakarta.ws.rs.ext.ParamConverter; import jakarta.ws.rs.ext.ParamConverterProvider; +import java.lang.annotation.Annotation; +import java.lang.reflect.Type; +import java.util.Objects; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.identifiable.NamespacedStorageProvider; @@ -42,7 +42,9 @@ public ParamConverter getConverter(Class rawType, Type genericType, An } if (NamespacedId.class.isAssignableFrom(rawType)) { - return new IdPathParamConverter(IdUtil.createParser((Class>) rawType), namespacedStorageProvider()); + return new IdPathParamConverter( + IdUtil.createParser((Class>) rawType), + namespacedStorageProvider()); } throw new IllegalStateException("Unsupported Id-type %s".formatted(rawType)); @@ -67,8 +69,9 @@ public boolean equals(Object obj) { return false; } var that = (IdPathParamConverterProvider) obj; - return Objects.equals(this.metaStorage, that.metaStorage) && - Objects.equals(this.namespacedStorageProvider, that.namespacedStorageProvider); + return Objects.equals(this.metaStorage, that.metaStorage) && Objects.equals( + this.namespacedStorageProvider, + that.namespacedStorageProvider); } @Override @@ -78,13 +81,13 @@ public int hashCode() { @Override public String toString() { - return "IdPathParamConverterProvider[" + - "metaStorage=" + metaStorage + ", " + - "namespacedStorageProvider=" + namespacedStorageProvider + ']'; + return "IdPathParamConverterProvider[" + "metaStorage=" + metaStorage + ", " + "namespacedStorageProvider=" + namespacedStorageProvider + ']'; } - public record IdPathParamConverter, STORAGE>(IdUtil.Parser parser, STORAGE storage) implements ParamConverter { + public record IdPathParamConverter, STORAGE>( + IdUtil.Parser parser, + STORAGE storage) implements ParamConverter { @Override public T fromString(String value) { diff --git a/backend/src/main/java/com/bakdata/conquery/io/jersey/LocaleFilter.java b/backend/src/main/java/com/bakdata/conquery/io/jersey/LocaleFilter.java index b78b96a88c8..84ff289f721 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jersey/LocaleFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jersey/LocaleFilter.java @@ -1,12 +1,11 @@ package com.bakdata.conquery.io.jersey; -import java.io.IOException; -import java.util.Locale; - import jakarta.servlet.http.HttpServletRequest; import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.container.ContainerRequestFilter; import jakarta.ws.rs.core.Context; +import java.io.IOException; +import java.util.Locale; import com.bakdata.conquery.models.i18n.I18n; @@ -17,10 +16,10 @@ */ public class LocaleFilter implements ContainerRequestFilter { - + @Context private HttpServletRequest request; - + @Override public void filter(ContainerRequestContext requestContext) throws IOException { // Use jetty context here because requestContext.getLanguage() does not work diff --git a/backend/src/main/java/com/bakdata/conquery/io/jersey/MessagePackProvider.java b/backend/src/main/java/com/bakdata/conquery/io/jersey/MessagePackProvider.java index 548cdb09829..186c0b3ed88 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jersey/MessagePackProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jersey/MessagePackProvider.java @@ -1,11 +1,5 @@ package com.bakdata.conquery.io.jersey; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.lang.annotation.Annotation; -import java.lang.reflect.Type; - import jakarta.ws.rs.Consumes; import jakarta.ws.rs.Produces; import jakarta.ws.rs.WebApplicationException; @@ -14,8 +8,12 @@ import jakarta.ws.rs.ext.MessageBodyReader; import jakarta.ws.rs.ext.MessageBodyWriter; import jakarta.ws.rs.ext.Provider; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.annotation.Annotation; +import java.lang.reflect.Type; -import com.bakdata.conquery.io.jackson.Jackson; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.RequiredArgsConstructor; @@ -38,7 +36,14 @@ public long getSize(T t, Class type, Type genericType, Annotation[] annotatio } @Override - public void writeTo(T t, Class type, Type genericType, Annotation[] annotations, MediaType mediaType, MultivaluedMap httpHeaders, OutputStream entityStream) throws IOException, WebApplicationException { + public void writeTo( + T t, + Class type, + Type genericType, + Annotation[] annotations, + MediaType mediaType, + MultivaluedMap httpHeaders, + OutputStream entityStream) throws IOException, WebApplicationException { objectMapper.writeValue(entityStream, t); entityStream.flush(); } @@ -49,9 +54,15 @@ public boolean isReadable(Class type, Type genericType, Annotation[] annotati } @Override - public T readFrom(Class type, Type genericType, Annotation[] annotations, MediaType mediaType, MultivaluedMap httpHeaders, InputStream entityStream) throws IOException, WebApplicationException { + public T readFrom( + Class type, + Type genericType, + Annotation[] annotations, + MediaType mediaType, + MultivaluedMap httpHeaders, + InputStream entityStream) throws IOException, WebApplicationException { return objectMapper.readValue(entityStream, type); } - - -} \ No newline at end of file + + +} diff --git a/backend/src/main/java/com/bakdata/conquery/io/jersey/RESTServer.java b/backend/src/main/java/com/bakdata/conquery/io/jersey/RESTServer.java index b32fb8d81cf..54c056c9cd9 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jersey/RESTServer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jersey/RESTServer.java @@ -39,10 +39,11 @@ public static void configure(ConqueryConfig config, ResourceConfig jersey) { // default Dropwizard's exception mappers jersey.register(new ConqueryErrorExceptionMapper()); jersey.register(ConqueryJsonExceptionMapper.class); - jersey.register(new LoggingExceptionMapper() {}); + jersey.register(new LoggingExceptionMapper() { + }); jersey.register(new EarlyEofExceptionMapper()); //allow cross origin - if(config.getApi().isAllowCORSRequests()) { + if (config.getApi().isAllowCORSRequests()) { jersey.register(CORSResponseFilter.class); jersey.register(new CORSPreflightRequestFilter()); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/jetty/CORSPreflightRequestFilter.java b/backend/src/main/java/com/bakdata/conquery/io/jetty/CORSPreflightRequestFilter.java index fccd44a4097..434058bf705 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jetty/CORSPreflightRequestFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jetty/CORSPreflightRequestFilter.java @@ -1,13 +1,13 @@ package com.bakdata.conquery.io.jetty; -import java.io.IOException; - -import com.google.common.net.HttpHeaders; import jakarta.annotation.Priority; import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.container.ContainerRequestFilter; import jakarta.ws.rs.container.PreMatching; import jakarta.ws.rs.core.Response; +import java.io.IOException; + +import com.google.common.net.HttpHeaders; import org.eclipse.jetty.http.HttpMethod; @PreMatching @@ -29,7 +29,7 @@ public void filter(ContainerRequestContext requestContext) throws IOException { if (requestContext.getHeaderString(HttpHeaders.ACCESS_CONTROL_REQUEST_METHOD) == null) { return; } - + // Preflight are not authenticated, so answer right away with an 204 // (https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/OPTIONS) requestContext.abortWith(Response.noContent().build()); diff --git a/backend/src/main/java/com/bakdata/conquery/io/jetty/CORSResponseFilter.java b/backend/src/main/java/com/bakdata/conquery/io/jetty/CORSResponseFilter.java index fdcb052738f..ab4461527f3 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jetty/CORSResponseFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jetty/CORSResponseFilter.java @@ -9,7 +9,10 @@ public class CORSResponseFilter implements ContainerResponseFilter { @Override public void filter(ContainerRequestContext requestContext, ContainerResponseContext rCtx) { rCtx.getHeaders().add("Access-Control-Allow-Origin", "*"); - rCtx.getHeaders().add("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept, Authorization, If-None-Match"); + rCtx.getHeaders() + .add( + "Access-Control-Allow-Headers", + "Origin, X-Requested-With, Content-Type, Accept, Authorization, If-None-Match"); rCtx.getHeaders().add("Access-Control-Allow-Methods", "GET, POST, PATCH,DELETE, PUT"); rCtx.getHeaders().add("Access-Control-Expose-Headers", "ETag"); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/jetty/CachingFilter.java b/backend/src/main/java/com/bakdata/conquery/io/jetty/CachingFilter.java index fa157ff9f16..11a1a5cb107 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jetty/CachingFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jetty/CachingFilter.java @@ -1,23 +1,24 @@ package com.bakdata.conquery.io.jetty; -import com.google.common.net.HttpHeaders; import jakarta.ws.rs.HttpMethod; import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.container.ContainerResponseContext; import jakarta.ws.rs.container.ContainerResponseFilter; +import com.google.common.net.HttpHeaders; + public class CachingFilter implements ContainerResponseFilter { @Override public void filter(ContainerRequestContext requestContext, ContainerResponseContext rCtx) { - + //if request where caching would make sense - if(requestContext.getMethod().equals(HttpMethod.GET)) { + if (requestContext.getMethod().equals(HttpMethod.GET)) { rCtx.getHeaders().add(HttpHeaders.CACHE_CONTROL, "private"); rCtx.getHeaders().add(HttpHeaders.CACHE_CONTROL, "must-revalidate"); - + //if no etag that means we actually want the browser to cache - if(rCtx.getEntityTag()==null) { + if (rCtx.getEntityTag() == null) { rCtx.getHeaders().add(HttpHeaders.CACHE_CONTROL, "no-cache"); rCtx.getHeaders().add(HttpHeaders.CACHE_CONTROL, "no-store"); rCtx.getHeaders().add(HttpHeaders.PRAGMA, "no-cache"); @@ -26,4 +27,4 @@ public void filter(ContainerRequestContext requestContext, ContainerResponseCont } } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/io/jetty/ConqueryErrorExceptionMapper.java b/backend/src/main/java/com/bakdata/conquery/io/jetty/ConqueryErrorExceptionMapper.java index 1c14b26a117..d429cad413b 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jetty/ConqueryErrorExceptionMapper.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jetty/ConqueryErrorExceptionMapper.java @@ -14,22 +14,20 @@ public class ConqueryErrorExceptionMapper implements ExceptionMapper { private final Provider requestContext; - + @Override public Response toResponse(ConstraintViolationException exception) { - log.warn("Json Validation error on {} {}:\nproblem:{}", + log.warn( + "Json Validation error on {} {}:\nproblem:{}", requestContext.get().getMethod(), requestContext.get().getRequestUri(), - Joiner.on('\n').join(Iterables.transform(exception.getConstraintViolations(), this::constraintMessageBuilder)) - ); + Joiner.on('\n') + .join( + Iterables.transform(exception.getConstraintViolations(), this::constraintMessageBuilder)) + ); return Response.status(422) - .type(MediaType.APPLICATION_JSON_TYPE) - .entity(new ErrorMessage(422, Joiner.on(" AND ").join(Iterables.transform(exception.getConstraintViolations(), this::constraintMessageBuilder)))) - .build(); + .type(MediaType.APPLICATION_JSON_TYPE) + .entity( + new ErrorMessage( + 422, + Joiner.on(" AND ") + .join( + Iterables.transform(exception.getConstraintViolations(), this::constraintMessageBuilder)))) + .build(); } - + private String constraintMessageBuilder(ConstraintViolation violation) { return violation.getPropertyPath().toString() + " " + violation.getMessage(); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/jetty/NoSuchElementExceptionMapper.java b/backend/src/main/java/com/bakdata/conquery/io/jetty/NoSuchElementExceptionMapper.java index 6ba1c87a523..e80ba1f7d21 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jetty/NoSuchElementExceptionMapper.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jetty/NoSuchElementExceptionMapper.java @@ -1,9 +1,9 @@ package com.bakdata.conquery.io.jetty; -import java.util.NoSuchElementException; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.ext.ExceptionMapper; +import java.util.NoSuchElementException; import lombok.extern.slf4j.Slf4j; @@ -12,6 +12,10 @@ public class NoSuchElementExceptionMapper implements ExceptionMapper { WriteFuture send(MESSAGE message); + void trySend(MESSAGE message); + @JsonIgnore SocketAddress getRemoteAddress(); + void awaitClose(); + @JsonIgnore boolean isConnected(); - + public static interface Transforming extends MessageSender { @JsonIgnore MessageSender getMessageParent(); + TARGET transform(MESSAGE message); - + @Override default WriteFuture send(MESSAGE message) { return getMessageParent().send(transform(message)); } - + @Override default void trySend(MESSAGE message) { getMessageParent().trySend(transform(message)); } - + @Override default SocketAddress getRemoteAddress() { return getMessageParent().getRemoteAddress(); } - + @Override default void awaitClose() { getMessageParent().awaitClose(); } - + @Override default boolean isConnected() { return getMessageParent().isConnected(); } } - + @AllArgsConstructor @ToString(of = "session") public static abstract class Simple> implements MessageSender { - - @JsonIgnore @Setter @Getter @NonNull + + @JsonIgnore + @Setter + @Getter + @NonNull protected NetworkSession session; - + @Override public WriteFuture send(MESSAGE message) { return session.send(message); } - + @Override public void trySend(MESSAGE message) { session.trySend(message); } - + @Override public SocketAddress getRemoteAddress() { return session.getRemoteAddress(); } - + @Override public void awaitClose() { session.awaitClose(); } - + @Override public boolean isConnected() { return session.isConnected(); diff --git a/backend/src/main/java/com/bakdata/conquery/io/mina/NetworkSession.java b/backend/src/main/java/com/bakdata/conquery/io/mina/NetworkSession.java index baf3138afe5..51315b794fc 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/mina/NetworkSession.java +++ b/backend/src/main/java/com/bakdata/conquery/io/mina/NetworkSession.java @@ -44,16 +44,20 @@ public WriteFuture send(final NetworkMessage message) { while (!queuedMessages.offer(message, 2, TimeUnit.MINUTES)) { logWaitingMessages(message); } - } - catch (InterruptedException e) { + } catch (InterruptedException e) { log.error("Unexpected interruption, while trying to queue: {}", message, e); return DefaultWriteFuture.newNotWrittenFuture(session, e); } WriteFuture future = session.write(message); - + future.addListener(f -> { if (f instanceof WriteFuture writeFuture && !writeFuture.isWritten()) { - log.error("Could not write message: {} ({} -> {})", message, session.getLocalAddress(), session.getRemoteAddress(), writeFuture.getException()); + log.error( + "Could not write message: {} ({} -> {})", + message, + session.getLocalAddress(), + session.getRemoteAddress(), + writeFuture.getException()); } }); future.addListener(f -> queuedMessages.remove(message)); @@ -65,11 +69,11 @@ private void logWaitingMessages(NetworkMessage message) { final String waiting; if (log.isTraceEnabled()) { waiting = new ArrayList<>(queuedMessages).stream() - .map(Objects::toString) - .map(NetworkSession::shorten) - .collect(Collectors.joining("\n\t\t- ")); - } - else { + .map(Objects::toString) + .map( + NetworkSession::shorten) + .collect(Collectors.joining("\n\t\t- ")); + } else { waiting = "%s messages".formatted(queuedMessages.size()); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/mina/jackson/AsyncJacksonProtocolFilter.java b/backend/src/main/java/com/bakdata/conquery/io/mina/jackson/AsyncJacksonProtocolFilter.java index db1763e7c6a..12a9083618c 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/mina/jackson/AsyncJacksonProtocolFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/io/mina/jackson/AsyncJacksonProtocolFilter.java @@ -52,7 +52,10 @@ public class AsyncJacksonProtocolFilter extends IoFilterAdapter { private static final DataSize PIPED_IO_BUFFER_SIZE = DataSize.kibibytes(2 * 64); - private static final Thread.Builder THREAD_BUILDER = Thread.ofVirtual().name("%s#reader".formatted(AsyncJacksonProtocolFilter.class.getSimpleName()), 0); + private static final Thread.Builder THREAD_BUILDER = Thread.ofVirtual() + .name( + "%s#reader".formatted(AsyncJacksonProtocolFilter.class.getSimpleName()), + 0); private static final AttributeKey READER = new AttributeKey(AsyncJacksonProtocolFilter.class, "reader"); private static final AttributeKey SPILL_BUFFER = new AttributeKey(AsyncJacksonProtocolFilter.class, "spill"); @@ -146,7 +149,10 @@ public void messageReceived(NextFilter nextFilter, IoSession session, Object mes } @Nullable - private AsyncReader getOrCreateAsyncReader(NextFilter nextFilter, IoSession session, IoBuffer buffer) throws IOException { + private AsyncReader getOrCreateAsyncReader( + NextFilter nextFilter, + IoSession session, + IoBuffer buffer) throws IOException { final AsyncReader asyncReader = getSessionReader(session); if (asyncReader != null) { @@ -163,8 +169,7 @@ private AsyncReader getOrCreateAsyncReader(NextFilter nextFilter, IoSession sess if (sessionHasSpillBuffer(session)) { bufferLength = lengthWithSpillBuffer(session, buffer); - } - else { + } else { // It's simply the prefix bufferLength = buffer.getInt(); } @@ -223,9 +228,15 @@ private static void createSpillBuffer(IoBuffer buffer, IoSession session) throws private static int lengthWithSpillBuffer(IoSession session, IoBuffer buffer) throws IOException { final IoBuffer spillBuffer = setSessionSpillBuffer(session, null); - log.trace("Found existing spill-buffer {}, incoming {}", DataSize.bytes(spillBuffer.position()), DataSize.bytes(buffer.limit())); + log.trace( + "Found existing spill-buffer {}, incoming {}", + DataSize.bytes(spillBuffer.position()), + DataSize.bytes(buffer.limit())); - buffer.getSlice(Integer.BYTES - spillBuffer.position()).asInputStream().transferTo(spillBuffer.asOutputStream()); + buffer.getSlice(Integer.BYTES - spillBuffer.position()) + .asInputStream() + .transferTo( + spillBuffer.asOutputStream()); spillBuffer.flip(); @@ -273,7 +284,11 @@ public void filterWrite(NextFilter nextFilter, IoSession session, WriteRequest w mapper.writerFor(NetworkMessage.class).writeValue(outputStream, message); } - log.trace("FINISHED Encoding message in {}. Buffer size: {}. Message: {}", stopwatch, DataSize.bytes(buf.remaining()), message); + log.trace( + "FINISHED Encoding message in {}. Buffer size: {}. Message: {}", + stopwatch, + DataSize.bytes(buf.remaining()), + message); buf.putInt(initialPosition, buf.position() - initialPosition - Integer.BYTES); buf.flip(); diff --git a/backend/src/main/java/com/bakdata/conquery/io/mina/jackson/AsyncPublisher.java b/backend/src/main/java/com/bakdata/conquery/io/mina/jackson/AsyncPublisher.java index fc1fe0b85b5..9c8d5371c47 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/mina/jackson/AsyncPublisher.java +++ b/backend/src/main/java/com/bakdata/conquery/io/mina/jackson/AsyncPublisher.java @@ -72,12 +72,10 @@ public void publish() { log.trace("RECEIVED {} from {}", reader, reader); reader.handleMessage(session); - } - catch (InterruptedException e) { + } catch (InterruptedException e) { Thread.interrupted(); log.trace("Interrupted", e); - } - catch (ExecutionException e) { + } catch (ExecutionException e) { log.error("Failed to get reader", e); } } @@ -90,8 +88,7 @@ public record FinishedMessage(IoFilter.NextFilter nextFilter, NetworkMessage net public void handleMessage(IoSession session) { try { nextFilter.messageReceived(session, networkMessage); - } - catch (Exception e) { + } catch (Exception e) { log.error("{} FAILED to deliver message {}", session, networkMessage, e); } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/mina/jackson/AsyncReader.java b/backend/src/main/java/com/bakdata/conquery/io/mina/jackson/AsyncReader.java index 2bbf0b36ae4..b2d689ed87f 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/mina/jackson/AsyncReader.java +++ b/backend/src/main/java/com/bakdata/conquery/io/mina/jackson/AsyncReader.java @@ -83,8 +83,7 @@ public void read() { log.trace("FINISHED parsing {} within {}", result, timer); futureMessage.complete(new AsyncPublisher.FinishedMessage(nextFilter, result)); - } - catch (IOException e) { + } catch (IOException e) { futureMessage.completeExceptionally(e); log.error("Something went wrong.", e); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/ExternalExecutionInfo.java b/backend/src/main/java/com/bakdata/conquery/io/result/ExternalExecutionInfo.java index 2059a65658d..3d730244a4f 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/ExternalExecutionInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/ExternalExecutionInfo.java @@ -1,10 +1,10 @@ package com.bakdata.conquery.io.result; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.UriBuilder; import java.util.List; import java.util.function.Function; import java.util.stream.Stream; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.io.external.form.ExternalFormBackendApi; diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/ResultRender/ResultRendererProvider.java b/backend/src/main/java/com/bakdata/conquery/io/result/ResultRender/ResultRendererProvider.java index 8623bfc0269..5f74021a06c 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/ResultRender/ResultRendererProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/ResultRender/ResultRendererProvider.java @@ -1,9 +1,9 @@ package com.bakdata.conquery.io.result.ResultRender; +import jakarta.ws.rs.core.UriBuilder; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.util.Collection; -import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.commands.ManagerNode; @@ -25,8 +25,10 @@ public interface ResultRendererProvider { * @param allProviders A flag that should override internal "hide-this-url" flags. * @return An Optional with the url or an empty optional. */ - Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) - throws MalformedURLException, URISyntaxException; + Collection generateResultURLs( + ManagedExecution exec, + UriBuilder uriBuilder, + boolean allProviders) throws MalformedURLException, URISyntaxException; void registerResultResource(DropwizardResourceConfig environment, ManagerNode manager); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java b/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java index 3b2a03695db..3d689a6d53f 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java @@ -2,13 +2,13 @@ import static com.bakdata.conquery.models.auth.AuthorizationHelper.authorizeDownloadDatasets; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.util.Locale; import jakarta.ws.rs.BadRequestException; import jakarta.ws.rs.core.HttpHeaders; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.Locale; import com.bakdata.conquery.models.auth.entities.Subject; import com.bakdata.conquery.models.auth.permissions.Ability; @@ -36,12 +36,20 @@ String getHeaderValue() { } - public static Response makeResponseWithFileName(Response.ResponseBuilder response, String filename, MediaType mediaType, ContentDispositionOption disposition) { + public static Response makeResponseWithFileName( + Response.ResponseBuilder response, + String filename, + MediaType mediaType, + ContentDispositionOption disposition) { response.header(HttpHeaders.CONTENT_TYPE, mediaType); if (!(Strings.isNullOrEmpty(filename) || filename.isBlank())) { // Set filename from filename if the filename was set, otherwise the browser will name the file according to the request path - response.header("Content-Disposition", String.format( - "%s; filename=\"%s\"", disposition.getHeaderValue(), FileUtil.makeSafeFileName(filename))); + response.header( + "Content-Disposition", + String.format( + "%s; filename=\"%s\"", + disposition.getHeaderValue(), + FileUtil.makeSafeFileName(filename))); } return response.build(); } @@ -54,8 +62,7 @@ public static Charset determineCharset(String userAgent, String queryCharset) { if (queryCharset != null) { try { return Charset.forName(queryCharset); - } - catch (Exception e) { + } catch (Exception e) { log.warn("Unable to map '{}' to a charset.", queryCharset); } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java index c8a4309082b..721b8e809fe 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java @@ -42,43 +42,62 @@ public class ArrowRenderer { public static void renderToStream( - Function writerProducer, - PrintSettings printSettings, - ArrowConfig arrowConfig, - List idHeaders, - List resultInfo, - Stream results, - PrinterFactory printerFactory) throws IOException { - - final List fields = ArrowUtil.generateFields(idHeaders, resultInfo, new UniqueNamer(printSettings), printSettings); + Function writerProducer, + PrintSettings printSettings, + ArrowConfig arrowConfig, + List idHeaders, + List resultInfo, + Stream results, + PrinterFactory printerFactory) throws IOException { + + final List fields = ArrowUtil.generateFields( + idHeaders, + resultInfo, + new UniqueNamer(printSettings), + printSettings); final VectorSchemaRoot root = VectorSchemaRoot.create(new Schema(fields, null), ROOT_ALLOCATOR); // Build separate pipelines for id and value, as they have different sources but the same target final RowConsumer[] idWriters = generateWriterPipeline(root, 0, idHeaders.size(), idHeaders); final RowConsumer[] valueWriter = generateWriterPipeline(root, idHeaders.size(), resultInfo.size(), resultInfo); - final List idPrinters = idHeaders.stream().map(info -> info.createPrinter(printerFactory, printSettings)).toList(); - final List resultPrinters = resultInfo.stream().map(info -> info.createPrinter(printerFactory, printSettings)).toList(); + final List idPrinters = idHeaders.stream() + .map( + info -> info.createPrinter(printerFactory, printSettings)) + .toList(); + final List resultPrinters = resultInfo.stream() + .map( + info -> info.createPrinter(printerFactory, printSettings)) + .toList(); // Write the data try (ArrowWriter writer = writerProducer.apply(root)) { - write(writer, root, idWriters, valueWriter, printSettings.getIdMapper(), idPrinters, resultPrinters, results, arrowConfig.getBatchSize()); + write( + writer, + root, + idWriters, + valueWriter, + printSettings.getIdMapper(), + idPrinters, + resultPrinters, + results, + arrowConfig.getBatchSize()); } } public static void write( - ArrowWriter writer, - VectorSchemaRoot root, - RowConsumer[] idWriters, - RowConsumer[] valueWriters, - PrintIdMapper idMapper, - List idPrinters, - List resultPrinters, - Stream results, - int batchSize) throws IOException { + ArrowWriter writer, + VectorSchemaRoot root, + RowConsumer[] idWriters, + RowConsumer[] valueWriters, + PrintIdMapper idMapper, + List idPrinters, + List resultPrinters, + Stream results, + int batchSize) throws IOException { Preconditions.checkArgument(batchSize > 0, "Batch size needs be larger than 0."); // TODO add time metric for writing @@ -107,10 +126,13 @@ public static void write( for (Object[] line : cer.listResultLines()) { currentEntityResultLine = line; - Preconditions.checkState(line.length == valueWriters.length, - "The number of value writers and values in a result line differs. Writers: %d Line: %d".formatted(valueWriters.length, - line.length - ) + Preconditions.checkState( + line.length == valueWriters.length, + "The number of value writers and values in a result line differs. Writers: %d Line: %d" + .formatted( + valueWriters.length, + line.length + ) ); for (int index = 0; index < idWriters.length; index++) { @@ -128,8 +150,7 @@ public static void write( printer = resultPrinters.get(index); // In this case, the printer normalizes and adjusts values. printed = printer.apply(value); - } - else { + } else { printed = null; } @@ -154,34 +175,34 @@ public static void write( root.clear(); batchCount++; } - } - catch (Exception e) { + } catch (Exception e) { throw new IllegalStateException( - String.format("Failed to write results. Failed with " - + "line-number=%s, row-number=%d, batch-number=%d, batch-line-number=%d, " - + "entity=%s, line-content=%s, " - + "value=%s, printed=%s, printer=%s", - lineCount, - currentRow, - batchCount, - batchLineCount, - currentEntityResult.getEntityId(), - Arrays.toString(currentEntityResultLine), - value, - printed, - printer - ), - e + String.format( + "Failed to write results. Failed with " + "line-number=%s, row-number=%d, batch-number=%d, batch-line-number=%d, " + "entity=%s, line-content=%s, " + "value=%s, printed=%s, printer=%s", + lineCount, + currentRow, + batchCount, + batchLineCount, + currentEntityResult.getEntityId(), + Arrays.toString(currentEntityResultLine), + value, + printed, + printer + ), + e ); - } - finally { + } finally { log.trace("Wrote {} batches of size {} (last batch might be smaller)", batchCount, batchSize); writer.end(); } } @NotNull - private static Object[] getPrintedExternalId(RowConsumer[] idWriters, PrintIdMapper idMapper, List printers, EntityResult cer) { + private static Object[] getPrintedExternalId( + RowConsumer[] idWriters, + PrintIdMapper idMapper, + List printers, + EntityResult cer) { final String[] externalId = idMapper.map(cer).getExternalId(); final Object[] printedExternalId = new String[externalId.length]; @@ -319,13 +340,18 @@ private static RowConsumer listVectorFiller(ListVector vector, RowConsumer neste } - public static RowConsumer[] generateWriterPipeline(VectorSchemaRoot root, int vectorOffset, int numVectors, List resultInfos) { + public static RowConsumer[] generateWriterPipeline( + VectorSchemaRoot root, + int vectorOffset, + int numVectors, + List resultInfos) { Preconditions.checkArgument(vectorOffset >= 0, "Offset was negative: %s", vectorOffset); Preconditions.checkArgument(numVectors >= 0, "Number of vectors was negative: %s", numVectors); final RowConsumer[] builder = new RowConsumer[numVectors]; - for (int vecI = vectorOffset; (vecI < root.getFieldVectors().size()) && (vecI < vectorOffset + numVectors); vecI++) { + for (int vecI = vectorOffset; (vecI < root.getFieldVectors() + .size()) && (vecI < vectorOffset + numVectors); vecI++) { final int pos = vecI - vectorOffset; final FieldVector vector = root.getVector(vecI); final ResultInfo resultInfo = resultInfos.get(pos); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java index 741885e9617..15b0be82c62 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java @@ -62,7 +62,10 @@ private static Field moneyField(@NonNull String uniqueName) { } private static Field floatField(@NonNull String uniqueName) { - return new Field(uniqueName, FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), null); + return new Field( + uniqueName, + FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + null); } private static Field dateField(@NonNull String uniqueName) { @@ -71,30 +74,35 @@ private static Field dateField(@NonNull String uniqueName) { private static Field dateRangeField(@NonNull String uniqueName) { return new Field( - uniqueName, - FieldType.nullable(ArrowType.Struct.INSTANCE), - List.of( - dateField("min"), - dateField("max") - )); + uniqueName, + FieldType.nullable(ArrowType.Struct.INSTANCE), + List.of( + dateField("min"), + dateField("max") + )); } private static Field listField(@NonNull String uniqueName, ResultType type) { final ResultType elementType = ((ResultType.ListT) type).getElementType(); final Field nestedField = fieldFor(elementType, uniqueName); - + return new Field(uniqueName, FieldType.nullable(ArrowType.List.INSTANCE), List.of(nestedField)); } - public static List generateFields(@NonNull List info, UniqueNamer collector, PrintSettings printSettings) { - return info.stream() - .map(i -> fieldFor(i.getType(), collector.getUniqueName(i, printSettings))) - .toList(); + public static List generateFields( + @NonNull List info, + UniqueNamer collector, + PrintSettings printSettings) { + return info.stream().map(i -> fieldFor(i.getType(), collector.getUniqueName(i, printSettings))).toList(); } @NotNull - public static List generateFields(List idHeaders, List resultInfo, UniqueNamer uniqueNamer, PrintSettings printSettings) { + public static List generateFields( + List idHeaders, + List resultInfo, + UniqueNamer uniqueNamer, + PrintSettings printSettings) { // Combine id and value Fields to one vector to build a schema List fields = new ArrayList<>(); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java index b6d71ad70ec..c682a2340c9 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java @@ -4,16 +4,16 @@ import static com.bakdata.conquery.resources.ResourceConstants.FILE_EXTENTION_ARROW_FILE; import static com.bakdata.conquery.resources.ResourceConstants.FILE_EXTENTION_ARROW_STREAM; +import jakarta.inject.Inject; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.StreamingOutput; import java.io.OutputStream; import java.nio.channels.Channels; import java.util.List; import java.util.Locale; import java.util.OptionalLong; import java.util.function.Function; -import jakarta.inject.Inject; -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.core.StreamingOutput; import com.bakdata.conquery.io.result.ResultUtil; import com.bakdata.conquery.models.auth.entities.Subject; @@ -58,30 +58,33 @@ public class ResultArrowProcessor { public Response createResultFile(Subject subject, ManagedExecutionId exec, boolean pretty, OptionalLong limit) { return getArrowResult( - (output) -> (root) -> new ArrowFileWriter(root, new DictionaryProvider.MapDictionaryProvider(), Channels.newChannel(output)), - subject, - (ManagedExecution & SingleTableResult) exec.resolve(), - datasetRegistry, - pretty, - FILE_EXTENTION_ARROW_FILE, - FILE_MEDIA_TYPE, - conqueryConfig, - arrowConfig, - limit + (output) -> (root) -> new ArrowFileWriter( + root, + new DictionaryProvider.MapDictionaryProvider(), + Channels.newChannel(output)), + subject, + (ManagedExecution & SingleTableResult) exec.resolve(), + datasetRegistry, + pretty, + FILE_EXTENTION_ARROW_FILE, + FILE_MEDIA_TYPE, + conqueryConfig, + arrowConfig, + limit ); } public static Response getArrowResult( - Function> writerProducer, - Subject subject, - E exec, - DatasetRegistry datasetRegistry, - boolean pretty, - String fileExtension, - MediaType mediaType, - ConqueryConfig config, - ArrowConfig arrowConfig, - OptionalLong limit + Function> writerProducer, + Subject subject, + E exec, + DatasetRegistry datasetRegistry, + boolean pretty, + String fileExtension, + MediaType mediaType, + ConqueryConfig config, + ArrowConfig arrowConfig, + OptionalLong limit ) { ConqueryMDC.setLocation(subject.getName()); @@ -109,43 +112,50 @@ public static Response getArrow CountingOutputStream countingOutputStream = new CountingOutputStream(output); try { ArrowRenderer.renderToStream( - writerProducer.apply(countingOutputStream), - settings, - arrowConfig, - resultInfosId, - resultInfosExec, - exec.streamResults(limit), - new ArrowResultPrinters() + writerProducer.apply(countingOutputStream), + settings, + arrowConfig, + resultInfosId, + resultInfosExec, + exec.streamResults(limit), + new ArrowResultPrinters() ); - } - catch (Exception e) { - throw new IllegalStateException("Failed streaming the result for execution %s requested by %s after %s".formatted(exec.getId(), - subject.getId(), - DataSize.bytes(countingOutputStream.getCount()) - ), - e + } catch (Exception e) { + throw new IllegalStateException( + "Failed streaming the result for execution %s requested by %s after %s".formatted( + exec.getId(), + subject.getId(), + DataSize.bytes(countingOutputStream.getCount()) + ), + e ); - } - finally { - log.trace("DONE downloading data for `{}` ({})", exec.getId(), DataSize.bytes(countingOutputStream.getCount())); + } finally { + log.trace( + "DONE downloading data for `{}` ({})", + exec.getId(), + DataSize.bytes(countingOutputStream.getCount())); } }; - return makeResponseWithFileName(Response.ok(out), String.join(".", exec.getLabelWithoutAutoLabelSuffix(), fileExtension), mediaType, ResultUtil.ContentDispositionOption.ATTACHMENT); + return makeResponseWithFileName( + Response.ok(out), + String.join(".", exec.getLabelWithoutAutoLabelSuffix(), fileExtension), + mediaType, + ResultUtil.ContentDispositionOption.ATTACHMENT); } public Response createResultStream(Subject subject, ManagedExecutionId exec, boolean pretty, OptionalLong limit) { return getArrowResult( - (output) -> (root) -> new ArrowStreamWriter(root, new DictionaryProvider.MapDictionaryProvider(), output), - subject, - ((ManagedExecution & SingleTableResult) exec.resolve()), - datasetRegistry, - pretty, - FILE_EXTENTION_ARROW_STREAM, - STREAM_MEDIA_TYPE, - conqueryConfig, - arrowConfig, - limit + (output) -> (root) -> new ArrowStreamWriter(root, new DictionaryProvider.MapDictionaryProvider(), output), + subject, + ((ManagedExecution & SingleTableResult) exec.resolve()), + datasetRegistry, + pretty, + FILE_EXTENTION_ARROW_STREAM, + STREAM_MEDIA_TYPE, + conqueryConfig, + arrowConfig, + limit ); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/RowConsumer.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/RowConsumer.java index 057a98306bd..49dff4b802c 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/RowConsumer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/RowConsumer.java @@ -3,11 +3,12 @@ @FunctionalInterface public interface RowConsumer { - /** - * Performs this operation on the given argument. - * - * @param t the input argument - */ - void accept(int rowNumber, Object value); + /** + * Performs this operation on the given argument. + * + * @param rowNumber current row number + * @param value object to be manipulated + */ + void accept(int rowNumber, Object value); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java index 4da2d4f7d8e..613e38f3984 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java @@ -26,10 +26,18 @@ public class CsvRenderer { private final CsvWriter writer; private final PrintSettings cfg; - public void toCSV(List idHeaders, List infos, Stream resultStream, PrintSettings printSettings, Charset charset) { + public void toCSV( + List idHeaders, + List infos, + Stream resultStream, + PrintSettings printSettings, + Charset charset) { UniqueNamer uniqNamer = new UniqueNamer(cfg); - final String[] headers = Stream.concat(idHeaders.stream(), infos.stream()).map(info -> uniqNamer.getUniqueName(info, printSettings)).toArray(String[]::new); + final String[] headers = Stream.concat(idHeaders.stream(), infos.stream()) + .map( + info -> uniqNamer.getUniqueName(info, printSettings)) + .toArray(String[]::new); writer.writeHeaders(headers); @@ -37,16 +45,21 @@ public void toCSV(List idHeaders, List infos, Stream infos, Stream results, PrintSettings printSettings, - PrinterFactory printerFactory) { - final Printer[] printers = infos.stream().map(info -> info.createPrinter(printerFactory, printSettings)).toArray(Printer[]::new); + PrintSettings cfg, + List infos, + Stream results, + PrintSettings printSettings, + PrinterFactory printerFactory) { + final Printer[] printers = infos.stream() + .map( + info -> info.createPrinter(printerFactory, printSettings)) + .toArray(Printer[]::new); results.map(result -> Pair.of(cfg.getIdMapper().map(result), result)) - .sorted(Map.Entry.comparingByKey()) - .forEach(res -> res - .getValue() - .streamValues() - .forEach(result -> printLine(res.getKey(), printers, result))); + .sorted( + Map.Entry.comparingByKey()) + .forEach( + res -> res.getValue().streamValues().forEach(result -> printLine(res.getKey(), printers, result))); } @@ -62,9 +75,10 @@ public void printLine(EntityPrintId entity, Printer[] printers, Object[] values) } try { writer.addValue(printers[i].apply(value)); - } - catch (Exception e) { - throw new IllegalStateException("Failed to print column %s of line %s".formatted(i, Arrays.deepToString(values)), e); + } catch (Exception e) { + throw new IllegalStateException( + "Failed to print column %s of line %s".formatted(i, Arrays.deepToString(values)), + e); } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java index 27b508ffe46..13338d93a1c 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java @@ -3,16 +3,16 @@ import static com.bakdata.conquery.io.result.ResultUtil.makeResponseWithFileName; import static com.bakdata.conquery.models.auth.AuthorizationHelper.authorizeDownloadDatasets; -import java.io.BufferedWriter; -import java.io.OutputStreamWriter; -import java.nio.charset.Charset; -import java.util.Locale; -import java.util.OptionalLong; import jakarta.inject.Inject; import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.StreamingOutput; +import java.io.BufferedWriter; +import java.io.OutputStreamWriter; +import java.nio.charset.Charset; +import java.util.Locale; +import java.util.OptionalLong; import com.bakdata.conquery.io.result.ResultUtil; import com.bakdata.conquery.models.auth.entities.Subject; @@ -37,7 +37,12 @@ public class ResultCsvProcessor { private final ConqueryConfig config; - public Response createResult(E exec, Subject subject, boolean pretty, Charset charset, OptionalLong limit) { + public Response createResult( + E exec, + Subject subject, + boolean pretty, + Charset charset, + OptionalLong limit) { ManagedExecutionId execId = exec.getId(); final Namespace namespace = exec.getNamespace(); @@ -63,23 +68,26 @@ public Response createResult(E final StreamingOutput out = os -> { try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(os, charset))) { final CsvRenderer renderer = new CsvRenderer(config.getCsv().createWriter(writer), settings); - renderer.toCSV(config.getIdColumns().getIdResultInfos(), exec.collectResultInfos(), exec.streamResults(limit), settings, charset); - } - catch (EofException e) { + renderer.toCSV( + config.getIdColumns().getIdResultInfos(), + exec.collectResultInfos(), + exec.streamResults(limit), + settings, + charset); + } catch (EofException e) { log.trace("User canceled download"); - } - catch (Exception e) { + } catch (Exception e) { throw new WebApplicationException("Failed to load result", e); - } - finally { + } finally { log.trace("FINISHED downloading {}", exec.getId()); } }; - return makeResponseWithFileName(Response.ok(out), - String.join(".", exec.getLabelWithoutAutoLabelSuffix(), ResourceConstants.FILE_EXTENTION_CSV), - new MediaType("text", "csv", charset.toString()), - ResultUtil.ContentDispositionOption.ATTACHMENT + return makeResponseWithFileName( + Response.ok(out), + String.join(".", exec.getLabelWithoutAutoLabelSuffix(), ResourceConstants.FILE_EXTENTION_CSV), + new MediaType("text", "csv", charset.toString()), + ResultUtil.ContentDispositionOption.ATTACHMENT ); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java index 72121d71149..35b32dfe43e 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java @@ -62,8 +62,12 @@ public ExcelRenderer(ExcelConfig config, PrintSettings settings) { } public void renderToStream( - List idHeaders, E exec, OutputStream outputStream, OptionalLong limit, PrintSettings printSettings, MetaStorage storage) - throws IOException { + List idHeaders, + E exec, + OutputStream outputStream, + OptionalLong limit, + PrintSettings printSettings, + MetaStorage storage) throws IOException { final List resultInfosExec = exec.getResultInfos(); setMetaData(exec, storage); @@ -77,13 +81,16 @@ public void renderToStream( writeHeader(sheet, idHeaders, resultInfosExec, table, printSettings); - final int writtenLines = writeBody(sheet, resultInfosExec, exec.streamResults(OptionalLong.of(limit.orElse(MAX_LINES))), new ExcelResultPrinters()); + final int writtenLines = writeBody( + sheet, + resultInfosExec, + exec.streamResults(OptionalLong.of(limit.orElse(MAX_LINES))), + new ExcelResultPrinters()); postProcessTable(sheet, table, writtenLines, idHeaders.size()); workbook.write(outputStream); - } - finally { + } finally { workbook.dispose(); } @@ -93,7 +100,9 @@ public void renderToStream( * Include meta data in the xlsx such as the title, owner/author, tag and the name of this instance. */ private void setMetaData(E exec, MetaStorage metaStorage) { - final POIXMLProperties.CoreProperties coreProperties = workbook.getXSSFWorkbook().getProperties().getCoreProperties(); + final POIXMLProperties.CoreProperties coreProperties = workbook.getXSSFWorkbook() + .getProperties() + .getCoreProperties(); coreProperties.setTitle(exec.getLabelWithoutAutoLabelSuffix()); String creator = config.getApplicationName(); @@ -108,7 +117,9 @@ private void setMetaData(E exec coreProperties.setCreator(creator); coreProperties.setKeywords(String.join(" ", exec.getTags())); - final POIXMLProperties.ExtendedProperties extendedProperties = workbook.getXSSFWorkbook().getProperties().getExtendedProperties(); + final POIXMLProperties.ExtendedProperties extendedProperties = workbook.getXSSFWorkbook() + .getProperties() + .getExtendedProperties(); extendedProperties.setApplication(config.getApplicationName()); } @@ -136,10 +147,11 @@ private XSSFTable createTableEnvironment(ManagedExecution exec, SXSSFSheet sheet * Also autosize the columns according to the header width. */ private void writeHeader( - SXSSFSheet sheet, - List idHeaders, - List infos, - XSSFTable table, PrintSettings printSettings) { + SXSSFSheet sheet, + List idHeaders, + List infos, + XSSFTable table, + PrintSettings printSettings) { final CTTableColumns columns = table.getCTTable().addNewTableColumns(); columns.setCount(idHeaders.size() + infos.size()); @@ -183,18 +195,23 @@ private void writeHeader( } private int writeBody( - SXSSFSheet sheet, - List infos, - Stream resultLines, PrinterFactory printerFactory) { + SXSSFSheet sheet, + List infos, + Stream resultLines, + PrinterFactory printerFactory) { // Row 0 is the Header the data starts at 1 final AtomicInteger currentRow = new AtomicInteger(1); - final TypeWriter[] writers = - infos.stream().map(info -> writer(info.getType(), info.createPrinter(printerFactory, settings), settings)).toArray(TypeWriter[]::new); + final TypeWriter[] writers = infos.stream() + .map( + info -> writer(info.getType(), info.createPrinter(printerFactory, settings), settings)) + .toArray( + TypeWriter[]::new); final PrintIdMapper idMapper = settings.getIdMapper(); - final int writtenLines = resultLines.mapToInt(l -> writeRowsForEntity(infos, l, currentRow, sheet, writers, idMapper)).sum(); + final int writtenLines = resultLines.mapToInt( + l -> writeRowsForEntity(infos, l, currentRow, sheet, writers, idMapper)).sum(); // The result was shorter than the number of rows to track, so we auto size here explicitly if (writtenLines < config.getLastRowToAutosize()) { @@ -246,12 +263,12 @@ private static TypeWriter writer(ResultType type, Printer printer, PrintSettings * Writes the result lines for each entity. */ private int writeRowsForEntity( - List infos, - EntityResult internalRow, - final AtomicInteger currentRow, - SXSSFSheet sheet, - TypeWriter[] writers, - PrintIdMapper idMapper) { + List infos, + EntityResult internalRow, + final AtomicInteger currentRow, + SXSSFSheet sheet, + TypeWriter[] writers, + PrintIdMapper idMapper) { final String[] ids = idMapper.map(internalRow).getExternalId(); @@ -337,11 +354,17 @@ public static void writeIntegerCell(Object value, Cell cell, Printer printer, Ma cell.setCellStyle(styles.get(ExcelConfig.INTEGER_STYLE)); } - public static void writeMoneyCell(Object valueRaw, Cell cell, Printer printer, PrintSettings settings, Map styles) { + public static void writeMoneyCell( + Object valueRaw, + Cell cell, + Printer printer, + PrintSettings settings, + Map styles) { final BigDecimal value = (BigDecimal) printer.apply(valueRaw); - final CellStyle currencyStyle = styles.get(ExcelConfig.CURRENCY_STYLE_PREFIX + settings.getCurrency().getCurrencyCode()); + final CellStyle currencyStyle = styles.get( + ExcelConfig.CURRENCY_STYLE_PREFIX + settings.getCurrency().getCurrencyCode()); if (currencyStyle == null) { // Print as cents or whatever the minor currency unit is cell.setCellValue(value.movePointRight(settings.getCurrency().getDefaultFractionDigits()).intValue()); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java index 876a08c2cf9..2ee1d6fcc19 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java @@ -2,12 +2,12 @@ import static com.bakdata.conquery.io.result.ResultUtil.makeResponseWithFileName; -import java.util.Locale; -import java.util.OptionalLong; import jakarta.inject.Inject; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.StreamingOutput; +import java.util.Locale; +import java.util.OptionalLong; import com.bakdata.conquery.io.result.ResultUtil; import com.bakdata.conquery.io.storage.MetaStorage; @@ -33,7 +33,9 @@ public class ResultExcelProcessor { // Media type according to https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types - public static final MediaType MEDIA_TYPE = new MediaType("application", "vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + public static final MediaType MEDIA_TYPE = new MediaType( + "application", + "vnd.openxmlformats-officedocument.spreadsheetml.sheet"); private final MetaStorage metaStorage; private final DatasetRegistry datasetRegistry; @@ -41,7 +43,11 @@ public class ResultExcelProcessor { private final ExcelConfig excelConfig; - public Response createResult(Subject subject, ManagedExecutionId execId, boolean pretty, OptionalLong limit) { + public Response createResult( + Subject subject, + ManagedExecutionId execId, + boolean pretty, + OptionalLong limit) { ConqueryMDC.setLocation(subject.getName()); @@ -52,7 +58,11 @@ public Response createResult(Su ResultUtil.authorizeExecutable(subject, exec); final Namespace namespace = datasetRegistry.get(exec.getDataset()); - final IdPrinter idPrinter = IdColumnUtil.getIdPrinter(subject, exec, namespace, conqueryConfig.getIdColumns().getIds()); + final IdPrinter idPrinter = IdColumnUtil.getIdPrinter( + subject, + exec, + namespace, + conqueryConfig.getIdColumns().getIds()); final Locale locale = I18n.LOCALE.get(); final PrintSettings settings = new PrintSettings(pretty, locale, conqueryConfig, idPrinter::createId, null); @@ -60,11 +70,21 @@ public Response createResult(Su final ExcelRenderer excelRenderer = new ExcelRenderer(excelConfig, settings); final StreamingOutput out = output -> { - excelRenderer.renderToStream(conqueryConfig.getIdColumns().getIdResultInfos(), exec, output, limit, settings, metaStorage); + excelRenderer.renderToStream( + conqueryConfig.getIdColumns().getIdResultInfos(), + exec, + output, + limit, + settings, + metaStorage); log.trace("FINISHED downloading {}", exec.getId()); }; - return makeResponseWithFileName(Response.ok(out), String.join(".", exec.getLabelWithoutAutoLabelSuffix(), ResourceConstants.FILE_EXTENTION_XLSX), MEDIA_TYPE, ResultUtil.ContentDispositionOption.ATTACHMENT); + return makeResponseWithFileName( + Response.ok(out), + String.join(".", exec.getLabelWithoutAutoLabelSuffix(), ResourceConstants.FILE_EXTENTION_XLSX), + MEDIA_TYPE, + ResultUtil.ContentDispositionOption.ATTACHMENT); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/json/ResultJsonDescriptionProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/json/ResultJsonDescriptionProcessor.java index 6a397b9197f..9bf96bf3c0b 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/json/ResultJsonDescriptionProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/json/ResultJsonDescriptionProcessor.java @@ -2,13 +2,13 @@ import static com.bakdata.conquery.io.result.ResultUtil.makeResponseWithFileName; -import java.io.BufferedWriter; -import java.io.OutputStreamWriter; import jakarta.inject.Inject; import jakarta.ws.rs.InternalServerErrorException; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.StreamingOutput; +import java.io.BufferedWriter; +import java.io.OutputStreamWriter; import com.bakdata.conquery.apiv1.query.QueryDescription; import com.bakdata.conquery.io.jackson.Jackson; @@ -39,25 +39,25 @@ public Response createResult(Subject subject, ManagedExecutionId execId) { final StreamingOutput out = os -> { try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(os))) { Jackson.MAPPER.copy() - .writerWithDefaultPrettyPrinter() - .forType(QueryDescription.class) - .writeValue(writer, exec.getSubmitted()); - } - catch (EofException e) { + .writerWithDefaultPrettyPrinter() + .forType(QueryDescription.class) + .writeValue( + writer, + exec.getSubmitted()); + } catch (EofException e) { log.trace("User canceled download"); - } - catch (Exception e) { + } catch (Exception e) { throw new InternalServerErrorException("Failed to load result", e); - } - finally { + } finally { log.trace("FINISHED downloading {}", execId); } }; - return makeResponseWithFileName(Response.ok(out), - String.join(".", exec.getLabelWithoutAutoLabelSuffix(), ResourceConstants.FILE_EXTENTION_JSON), - MediaType.APPLICATION_JSON_TYPE, - ResultUtil.ContentDispositionOption.ATTACHMENT + return makeResponseWithFileName( + Response.ok(out), + String.join(".", exec.getLabelWithoutAutoLabelSuffix(), ResourceConstants.FILE_EXTENTION_JSON), + MediaType.APPLICATION_JSON_TYPE, + ResultUtil.ContentDispositionOption.ATTACHMENT ); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ColumnConsumer.java b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ColumnConsumer.java index c133c3635a7..9edaab815c4 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ColumnConsumer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ColumnConsumer.java @@ -14,4 +14,4 @@ @FunctionalInterface public interface ColumnConsumer extends BiConsumer { -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java index 387eadbb091..6f9b476d822 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java @@ -56,31 +56,43 @@ public class EntityResultWriteSupport extends WriteSupport { * @param printSettings1 * @return the parquet schema */ - public static MessageType generateSchema(List idHeaders, List resultValueInfos, UniqueNamer uniqueNamer, PrintSettings printSettings1) { + public static MessageType generateSchema( + List idHeaders, + List resultValueInfos, + UniqueNamer uniqueNamer, + PrintSettings printSettings1) { /* Because Parquet Schemas rely on primitive types with logical annotations which are tedious to configure, we take the detour over the arrow schema. */ - final SchemaMapping schemaMapping = - new SchemaConverter().fromArrow(new Schema(ArrowUtil.generateFields(idHeaders, resultValueInfos, uniqueNamer, printSettings1))); + final SchemaMapping schemaMapping = new SchemaConverter().fromArrow( + new Schema(ArrowUtil.generateFields(idHeaders, resultValueInfos, uniqueNamer, printSettings1))); return schemaMapping.getParquetSchema(); } - private static List generateColumnConsumers(List idHeaders, List resultInfos) { + private static List generateColumnConsumers( + List idHeaders, + List resultInfos) { return Stream.concat(idHeaders.stream(), resultInfos.stream()) - .map(ResultInfo::getType) - .map(EntityResultWriteSupport::columnConsumerForType) - .toList(); + .map(ResultInfo::getType) + .map( + EntityResultWriteSupport::columnConsumerForType) + .toList(); } - private static List generateColumnPrinters(List idHeaders, List resultInfos, PrintSettings printSettings, PrinterFactory printerFactory) { + private static List generateColumnPrinters( + List idHeaders, + List resultInfos, + PrintSettings printSettings, + PrinterFactory printerFactory) { return Stream.concat(idHeaders.stream(), resultInfos.stream()) - .map(info -> info.createPrinter(printerFactory, printSettings)) - .toList(); + .map( + info -> info.createPrinter(printerFactory, printSettings)) + .toList(); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ParquetRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ParquetRenderer.java index a903133ff68..40632477fdb 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ParquetRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ParquetRenderer.java @@ -25,20 +25,18 @@ public class ParquetRenderer { public static void writeToStream( - OutputStream outputStream, - List idHeaders, - List resultInfo, - PrintSettings printSettings, - Stream results) throws IOException { + OutputStream outputStream, + List idHeaders, + List resultInfo, + PrintSettings printSettings, + Stream results) throws IOException { // Wrap the request output stream in an output file, so the parquet writer can consume it - final OutputFile outputFile = new StreamOutputFile(new PositionTrackingOutputStream(new CountingOutputStream(outputStream))); + final OutputFile outputFile = new StreamOutputFile( + new PositionTrackingOutputStream(new CountingOutputStream(outputStream))); - final ConqueryParquetWriterBuilder conqueryParquetWriterBuilder = - new ConqueryParquetWriterBuilder(outputFile) - .setIdHeaders(idHeaders) - .setResultInfo(resultInfo) - .setPrintSettings(printSettings); + final ConqueryParquetWriterBuilder conqueryParquetWriterBuilder = new ConqueryParquetWriterBuilder( + outputFile).setIdHeaders(idHeaders).setResultInfo(resultInfo).setPrintSettings(printSettings); try (final ParquetWriter parquetWriter = conqueryParquetWriterBuilder.build()) { @@ -46,7 +44,8 @@ public static void writeToStream( WORKAROUND: We need the conversion to SinglelineEntityResult here because a RecordConsumer only produces one line/record even if multiple messages are started. */ - Iterator resultIterator = results.flatMap(ParquetRenderer::convertToSingleLine).iterator(); + Iterator resultIterator = results.flatMap( + ParquetRenderer::convertToSingleLine).iterator(); while (resultIterator.hasNext()) { final EntityResult entityResult = resultIterator.next(); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java index 6479a0796cd..45bd9f8f1aa 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java @@ -2,12 +2,12 @@ import static com.bakdata.conquery.io.result.ResultUtil.makeResponseWithFileName; -import java.util.Locale; -import java.util.OptionalLong; import jakarta.inject.Inject; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.StreamingOutput; +import java.util.Locale; +import java.util.OptionalLong; import com.bakdata.conquery.io.result.ResultUtil; import com.bakdata.conquery.models.auth.entities.Subject; @@ -30,7 +30,8 @@ @Slf4j @RequiredArgsConstructor(onConstructor_ = {@Inject}) public class ResultParquetProcessor { - public static final MediaType PARQUET_MEDIA_TYPE = MediaType.valueOf(ResultParquetResource.PARQUET_MEDIA_TYPE_STRING); + public static final MediaType PARQUET_MEDIA_TYPE = MediaType.valueOf( + ResultParquetResource.PARQUET_MEDIA_TYPE_STRING); private final DatasetRegistry datasetRegistry; private final ConqueryConfig config; @@ -49,7 +50,11 @@ public Response createResultFile(Subject subject, ManagedExecutionId execId, boo final Namespace namespace = datasetRegistry.get(execution.getDataset()); - final IdPrinter idPrinter = IdColumnUtil.getIdPrinter(subject, execution, namespace, config.getIdColumns().getIds()); + final IdPrinter idPrinter = IdColumnUtil.getIdPrinter( + subject, + execution, + namespace, + config.getIdColumns().getIds()); final Locale locale = I18n.LOCALE.get(); final PrintSettings settings = new PrintSettings(pretty, locale, config, idPrinter::createId, null); @@ -58,20 +63,21 @@ public Response createResultFile(Subject subject, ManagedExecutionId execId, boo final SingleTableResult singleTableResult = (SingleTableResult) execution; ParquetRenderer.writeToStream( - output, - config.getIdColumns().getIdResultInfos(), - singleTableResult.collectResultInfos(), - settings, - singleTableResult.streamResults(limit) + output, + config.getIdColumns().getIdResultInfos(), + singleTableResult.collectResultInfos(), + settings, + singleTableResult.streamResults(limit) ); }; - return makeResponseWithFileName(Response.ok(out), - String.join(".", execution.getLabelWithoutAutoLabelSuffix(), ResourceConstants.FILE_EXTENTION_PARQUET), - PARQUET_MEDIA_TYPE, - ResultUtil.ContentDispositionOption.ATTACHMENT + return makeResponseWithFileName( + Response.ok(out), + String.join(".", execution.getLabelWithoutAutoLabelSuffix(), ResourceConstants.FILE_EXTENTION_PARQUET), + PARQUET_MEDIA_TYPE, + ResultUtil.ContentDispositionOption.ATTACHMENT ); } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/IdentifiableStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/IdentifiableStore.java index 387a5c2a6fb..26eba8deffe 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/IdentifiableStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/IdentifiableStore.java @@ -1,31 +1,31 @@ -package com.bakdata.conquery.io.storage; - -import com.bakdata.conquery.io.storage.xodus.stores.KeyIncludingStore; -import com.bakdata.conquery.models.identifiable.Identifiable; -import com.bakdata.conquery.models.identifiable.ids.Id; -import com.bakdata.conquery.util.functions.ThrowingConsumer; -import lombok.Getter; -import lombok.Setter; -import lombok.experimental.Accessors; - -/** - * Store for Identifiable values. - *

- * The {@link ThrowingConsumer}s can be used to reflect/model dependencies of the identifiable values inside the store. For example {@link com.bakdata.conquery.models.datasets.concepts.Concept} holds multiple {@link com.bakdata.conquery.models.datasets.concepts.Connector}s where a deletion of a concept requires the deletion of the Conncetors as well. {@link NamespacedStorageImpl} is the main user of those two methods and should be looked at if desired. - */ -@Accessors(fluent = true) -@Setter -@Getter -public class IdentifiableStore> extends KeyIncludingStore, VALUE> { - - - public IdentifiableStore(Store, VALUE> store) { - super(store); - } - - - @Override - protected Id extractKey(VALUE value) { - return (Id) value.getId(); - } -} +package com.bakdata.conquery.io.storage; + +import com.bakdata.conquery.io.storage.xodus.stores.KeyIncludingStore; +import com.bakdata.conquery.models.identifiable.Identifiable; +import com.bakdata.conquery.models.identifiable.ids.Id; +import com.bakdata.conquery.util.functions.ThrowingConsumer; +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; + +/** + * Store for Identifiable values. + *

+ * The {@link ThrowingConsumer}s can be used to reflect/model dependencies of the identifiable values inside the store. For example {@link com.bakdata.conquery.models.datasets.concepts.Concept} holds multiple {@link com.bakdata.conquery.models.datasets.concepts.Connector}s where a deletion of a concept requires the deletion of the Conncetors as well. {@link NamespacedStorageImpl} is the main user of those two methods and should be looked at if desired. + */ +@Accessors(fluent = true) +@Setter +@Getter +public class IdentifiableStore> extends KeyIncludingStore, VALUE> { + + + public IdentifiableStore(Store, VALUE> store) { + super(store); + } + + + @Override + protected Id extractKey(VALUE value) { + return (Id) value.getId(); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/ManagedStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/ManagedStore.java index 442b37d3548..eb0ed7db6af 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/ManagedStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/ManagedStore.java @@ -7,9 +7,11 @@ */ public interface ManagedStore { void loadData(); + void close() throws IOException; void clear(); + void removeStore(); void invalidateCache(); diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java index b9d9c4154b7..02af7d92522 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java @@ -1,215 +1,212 @@ -package com.bakdata.conquery.io.storage; - -import java.util.Objects; -import java.util.stream.Stream; - -import com.bakdata.conquery.io.jackson.Injectable; -import com.bakdata.conquery.io.jackson.MutableInjectableValues; -import com.bakdata.conquery.models.auth.entities.Group; -import com.bakdata.conquery.models.auth.entities.Role; -import com.bakdata.conquery.models.auth.entities.User; -import com.bakdata.conquery.models.config.StoreFactory; -import com.bakdata.conquery.models.execution.ManagedExecution; -import com.bakdata.conquery.models.forms.configs.FormConfig; -import com.bakdata.conquery.models.identifiable.ids.specific.FormConfigId; -import com.bakdata.conquery.models.identifiable.ids.specific.GroupId; -import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; -import com.bakdata.conquery.models.identifiable.ids.specific.RoleId; -import com.bakdata.conquery.models.identifiable.ids.specific.UserId; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; - -/** - * Access to persisted entities that are not namespace/dataset crucial (see {@link NamespacedStorageImpl}). - * All entities are loaded through a cache. The cache can be configured through the StoreFactory. - */ -@Slf4j -@RequiredArgsConstructor -public class MetaStorage implements ConqueryStorage, Injectable { - - private final StoreFactory storageFactory; - private IdentifiableStore executions; - private IdentifiableStore formConfigs; - private IdentifiableStore authUser; - private IdentifiableStore authRole; - private IdentifiableStore authGroup; - - public static MetaStorage getInjected(DeserializationContext ctxt) throws JsonMappingException { - return (MetaStorage) ctxt - .findInjectableValue(MetaStorage.class.getName(), null, null); - } - - @Override - public ImmutableList getStores() { - Preconditions.checkNotNull(authUser, "User storage was not created"); - Preconditions.checkNotNull(authRole, "Role storage was not created"); - Preconditions.checkNotNull(authGroup, "Group storage was not created"); - Preconditions.checkNotNull(executions, "Execution storage was not created"); - Preconditions.checkNotNull(formConfigs, "FormConfig storage was not created"); - - return ImmutableList.of( - authUser, - authRole, - authGroup, - executions, - formConfigs - ); - } - - public void openStores(ObjectMapper mapper) { - if (mapper != null) { - this.injectInto(mapper); - } - authUser = storageFactory.createUserStore("meta", mapper); - authRole = storageFactory.createRoleStore("meta", mapper); - authGroup = storageFactory.createGroupStore("meta", mapper); - // Executions depend on users - executions = storageFactory.createExecutionsStore("meta", mapper); - formConfigs = storageFactory.createFormConfigStore("meta", mapper); - - } - - // Executions - - public void addExecution(ManagedExecution query) { - executions.add(query); - } - - public ManagedExecution getExecution(ManagedExecutionId id) { - return executions.get(id); - } - - public Stream getAllExecutions() { - return executions.getAllKeys() - .map(executions::get) - .filter(Objects::nonNull); - } - - public Stream getAllExecutionIds() { - return executions.getAllKeys().map(ManagedExecutionId.class::cast); - } - - public synchronized void updateExecution(ManagedExecution query) { - executions.update(query); - } - - public synchronized void removeExecution(ManagedExecutionId id) { - executions.remove(id); - } - - // Groups - - public synchronized void addGroup(Group group) { - log.info("Adding group = {}", group.getId()); - authGroup.add(group); - } - - public Group getGroup(GroupId groupId) { - final Group group = authGroup.get(groupId); - log.trace("Requested group '{}' got: {}", groupId, group); - return group; - } - - public Stream getAllGroups() { - return authGroup.getAllKeys().map(authGroup::get); - } - - public void removeGroup(GroupId id) { - log.info("Removing group = {}", id); - authGroup.remove(id); - } - - public synchronized void updateGroup(Group group) { - log.info("Updating group = {}", group.getId()); - authGroup.update(group); - } - - // User - - public synchronized void addUser(User user) { - log.info("Adding user = {}", user.getId()); - authUser.add(user); - } - - public User getUser(UserId userId) { - final User user = authUser.get(userId); - log.trace("Requested user '{}' got: {}", userId, user); - return user; - } - - public Stream getAllUsers() { - return authUser.getAllKeys().map(authUser::get); - } - - public synchronized void removeUser(UserId userId) { - log.info("Removing user = {}", userId); - authUser.remove(userId); - } - - public synchronized void updateUser(User user) { - log.info("Updating user = {}", user.getId()); - authUser.update(user); - } - - // Roles - - public synchronized void addRole(Role role) { - authRole.add(role); - } - - public Role getRole(RoleId roleId) { - final Role role = authRole.get(roleId); - log.trace("Requested role '{}' got: {}", roleId, role); - return role; - } - - public Stream getAllRoles() { - return authRole.getAllKeys().map(authRole::get); - } - - public synchronized void removeRole(RoleId roleId) { - log.info("Removing role = {}", roleId); - authRole.remove(roleId); - } - - public synchronized void updateRole(Role role) { - log.info("Updating role = {}", role.getId()); - authRole.update(role); - } - - // FormConfigs - - public FormConfig getFormConfig(FormConfigId id) { - return formConfigs.get(id); - } - - public Stream getAllFormConfigs() { - return formConfigs.getAllKeys().map(formConfigs::get); - } - - public synchronized void removeFormConfig(FormConfigId id) { - formConfigs.remove(id); - } - - @SneakyThrows - public synchronized void updateFormConfig(FormConfig formConfig) { - formConfigs.update(formConfig); - } - - @SneakyThrows - public synchronized void addFormConfig(FormConfig formConfig) { - formConfigs.add(formConfig); - } - - @Override - public MutableInjectableValues inject(MutableInjectableValues values) { - return values.add(MetaStorage.class, this); - } - -} +package com.bakdata.conquery.io.storage; + +import java.util.Objects; +import java.util.stream.Stream; + +import com.bakdata.conquery.io.jackson.Injectable; +import com.bakdata.conquery.io.jackson.MutableInjectableValues; +import com.bakdata.conquery.models.auth.entities.Group; +import com.bakdata.conquery.models.auth.entities.Role; +import com.bakdata.conquery.models.auth.entities.User; +import com.bakdata.conquery.models.config.StoreFactory; +import com.bakdata.conquery.models.execution.ManagedExecution; +import com.bakdata.conquery.models.forms.configs.FormConfig; +import com.bakdata.conquery.models.identifiable.ids.specific.FormConfigId; +import com.bakdata.conquery.models.identifiable.ids.specific.GroupId; +import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; +import com.bakdata.conquery.models.identifiable.ids.specific.RoleId; +import com.bakdata.conquery.models.identifiable.ids.specific.UserId; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +/** + * Access to persisted entities that are not namespace/dataset crucial (see {@link NamespacedStorageImpl}). + * All entities are loaded through a cache. The cache can be configured through the StoreFactory. + */ +@Slf4j +@RequiredArgsConstructor +public class MetaStorage implements ConqueryStorage, Injectable { + + private final StoreFactory storageFactory; + private IdentifiableStore executions; + private IdentifiableStore formConfigs; + private IdentifiableStore authUser; + private IdentifiableStore authRole; + private IdentifiableStore authGroup; + + public static MetaStorage getInjected(DeserializationContext ctxt) throws JsonMappingException { + return (MetaStorage) ctxt.findInjectableValue(MetaStorage.class.getName(), null, null); + } + + @Override + public ImmutableList getStores() { + Preconditions.checkNotNull(authUser, "User storage was not created"); + Preconditions.checkNotNull(authRole, "Role storage was not created"); + Preconditions.checkNotNull(authGroup, "Group storage was not created"); + Preconditions.checkNotNull(executions, "Execution storage was not created"); + Preconditions.checkNotNull(formConfigs, "FormConfig storage was not created"); + + return ImmutableList.of( + authUser, + authRole, + authGroup, + executions, + formConfigs + ); + } + + public void openStores(ObjectMapper mapper) { + if (mapper != null) { + this.injectInto(mapper); + } + authUser = storageFactory.createUserStore("meta", mapper); + authRole = storageFactory.createRoleStore("meta", mapper); + authGroup = storageFactory.createGroupStore("meta", mapper); + // Executions depend on users + executions = storageFactory.createExecutionsStore("meta", mapper); + formConfigs = storageFactory.createFormConfigStore("meta", mapper); + + } + + // Executions + + public void addExecution(ManagedExecution query) { + executions.add(query); + } + + public ManagedExecution getExecution(ManagedExecutionId id) { + return executions.get(id); + } + + public Stream getAllExecutions() { + return executions.getAllKeys().map(executions::get).filter(Objects::nonNull); + } + + public Stream getAllExecutionIds() { + return executions.getAllKeys().map(ManagedExecutionId.class::cast); + } + + public synchronized void updateExecution(ManagedExecution query) { + executions.update(query); + } + + public synchronized void removeExecution(ManagedExecutionId id) { + executions.remove(id); + } + + // Groups + + public synchronized void addGroup(Group group) { + log.info("Adding group = {}", group.getId()); + authGroup.add(group); + } + + public Group getGroup(GroupId groupId) { + final Group group = authGroup.get(groupId); + log.trace("Requested group '{}' got: {}", groupId, group); + return group; + } + + public Stream getAllGroups() { + return authGroup.getAllKeys().map(authGroup::get); + } + + public void removeGroup(GroupId id) { + log.info("Removing group = {}", id); + authGroup.remove(id); + } + + public synchronized void updateGroup(Group group) { + log.info("Updating group = {}", group.getId()); + authGroup.update(group); + } + + // User + + public synchronized void addUser(User user) { + log.info("Adding user = {}", user.getId()); + authUser.add(user); + } + + public User getUser(UserId userId) { + final User user = authUser.get(userId); + log.trace("Requested user '{}' got: {}", userId, user); + return user; + } + + public Stream getAllUsers() { + return authUser.getAllKeys().map(authUser::get); + } + + public synchronized void removeUser(UserId userId) { + log.info("Removing user = {}", userId); + authUser.remove(userId); + } + + public synchronized void updateUser(User user) { + log.info("Updating user = {}", user.getId()); + authUser.update(user); + } + + // Roles + + public synchronized void addRole(Role role) { + authRole.add(role); + } + + public Role getRole(RoleId roleId) { + final Role role = authRole.get(roleId); + log.trace("Requested role '{}' got: {}", roleId, role); + return role; + } + + public Stream getAllRoles() { + return authRole.getAllKeys().map(authRole::get); + } + + public synchronized void removeRole(RoleId roleId) { + log.info("Removing role = {}", roleId); + authRole.remove(roleId); + } + + public synchronized void updateRole(Role role) { + log.info("Updating role = {}", role.getId()); + authRole.update(role); + } + + // FormConfigs + + public FormConfig getFormConfig(FormConfigId id) { + return formConfigs.get(id); + } + + public Stream getAllFormConfigs() { + return formConfigs.getAllKeys().map(formConfigs::get); + } + + public synchronized void removeFormConfig(FormConfigId id) { + formConfigs.remove(id); + } + + @SneakyThrows + public synchronized void updateFormConfig(FormConfig formConfig) { + formConfigs.update(formConfig); + } + + @SneakyThrows + public synchronized void addFormConfig(FormConfig formConfig) { + formConfigs.add(formConfig); + } + + @Override + public MutableInjectableValues inject(MutableInjectableValues values) { + return values.add(MetaStorage.class, this); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java index 148095b134f..0e3000e2f44 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java @@ -1,179 +1,179 @@ -package com.bakdata.conquery.io.storage; - -import java.util.Objects; -import java.util.stream.Stream; - -import com.bakdata.conquery.io.jackson.MutableInjectableValues; -import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore; -import com.bakdata.conquery.models.config.StoreFactory; -import com.bakdata.conquery.models.datasets.PreviewConfig; -import com.bakdata.conquery.models.datasets.concepts.StructureNode; -import com.bakdata.conquery.models.identifiable.ids.specific.InternToExternMapperId; -import com.bakdata.conquery.models.identifiable.ids.specific.SearchIndexId; -import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; -import com.bakdata.conquery.models.index.InternToExternMapper; -import com.bakdata.conquery.models.index.search.SearchIndex; -import com.bakdata.conquery.models.worker.WorkerToBucketsMap; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.ImmutableList; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class NamespaceStorage extends NamespacedStorageImpl { - - protected IdentifiableStore internToExternMappers; - protected IdentifiableStore searchIndexes; - protected SingletonStore idMapping; - protected SingletonStore structure; - protected SingletonStore preview; - protected SingletonStore workerToBuckets; - - - public NamespaceStorage(StoreFactory storageFactory, String pathName) { - super(storageFactory, pathName); - } - - @Override - public void openStores(ObjectMapper objectMapper) { - super.openStores(objectMapper); - - internToExternMappers = getStorageFactory().createInternToExternMappingStore(super.getPathName(), objectMapper); - searchIndexes = getStorageFactory().createSearchIndexStore(super.getPathName(), objectMapper); - idMapping = getStorageFactory().createIdMappingStore(super.getPathName(), objectMapper); - structure = getStorageFactory().createStructureStore(super.getPathName(), objectMapper); - workerToBuckets = getStorageFactory().createWorkerToBucketsStore(super.getPathName(), objectMapper); - preview = getStorageFactory().createPreviewStore(super.getPathName(), objectMapper); - } - - @Override - public ImmutableList getStores() { - return ImmutableList.of( - dataset, - - internToExternMappers, - searchIndexes, - - secondaryIds, - tables, - imports, - - // Concepts depend on internToExternMappers - concepts, - - preview, - idMapping, - structure, - workerToBuckets, - entity2Bucket - ); - } - - - // IdMapping - - public EntityIdMap getIdMapping() { - return idMapping.get(); - } - - public void updateIdMapping(EntityIdMap idMapping) { - this.idMapping.update(idMapping); - } - - // Bucket to Worker Assignment - - public void setWorkerToBucketsMap(WorkerToBucketsMap map) { - workerToBuckets.update(map); - } - - public WorkerToBucketsMap getWorkerBuckets() { - return workerToBuckets.get(); - } - - public int getNumberOfEntities() { - return entity2Bucket.count(); - } - - - public boolean containsEntity(String entity) { - return entity2Bucket.hasKey(entity); - } - - public void registerEntity(String entity, int bucket) { - entity2Bucket.update(entity, bucket); - } - - // Structure - - public StructureNode[] getStructure() { - return Objects.requireNonNullElseGet(structure.get(), () -> new StructureNode[0]); - } - - public void updateStructure(StructureNode[] structure) { - this.structure.update(structure); - } - - // InternToExternMappers - - public InternToExternMapper getInternToExternMapper(InternToExternMapperId id) { - return getInternToExternMapperFromStorage(id); - } - - private InternToExternMapper getInternToExternMapperFromStorage(InternToExternMapperId id) { - return internToExternMappers.get(id); - } - - public void addInternToExternMapper(InternToExternMapper internToExternMapper) { - internToExternMappers.add(internToExternMapper); - } - - public void removeInternToExternMapper(InternToExternMapperId id) { - internToExternMappers.remove(id); - } - - public Stream getInternToExternMappers() { - return internToExternMappers.getAll(); - } - - // SearchIndices - - public SearchIndex getSearchIndex(SearchIndexId id) { - return getSearchIndexFromStorage(id); - } - - private SearchIndex getSearchIndexFromStorage(SearchIndexId id) { - return searchIndexes.get(id); - } - - public void removeSearchIndex(SearchIndexId id) { - searchIndexes.remove(id); - } - - public void addSearchIndex(SearchIndex searchIndex) { - searchIndexes.add(searchIndex); - } - - public Stream getSearchIndices() { - return searchIndexes.getAll(); - } - - // PreviewConfig - - public PreviewConfig getPreviewConfig() { - return preview.get(); - } - - public void setPreviewConfig(PreviewConfig previewConfig){ - preview.update(previewConfig); - } - - public void removePreviewConfig() { - preview.remove(); - } - - // Utilities - - @Override - public MutableInjectableValues inject(MutableInjectableValues values) { - return super.inject(values).add(NamespaceStorage.class, this); - } -} +package com.bakdata.conquery.io.storage; + +import java.util.Objects; +import java.util.stream.Stream; + +import com.bakdata.conquery.io.jackson.MutableInjectableValues; +import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore; +import com.bakdata.conquery.models.config.StoreFactory; +import com.bakdata.conquery.models.datasets.PreviewConfig; +import com.bakdata.conquery.models.datasets.concepts.StructureNode; +import com.bakdata.conquery.models.identifiable.ids.specific.InternToExternMapperId; +import com.bakdata.conquery.models.identifiable.ids.specific.SearchIndexId; +import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; +import com.bakdata.conquery.models.index.InternToExternMapper; +import com.bakdata.conquery.models.index.search.SearchIndex; +import com.bakdata.conquery.models.worker.WorkerToBucketsMap; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class NamespaceStorage extends NamespacedStorageImpl { + + protected IdentifiableStore internToExternMappers; + protected IdentifiableStore searchIndexes; + protected SingletonStore idMapping; + protected SingletonStore structure; + protected SingletonStore preview; + protected SingletonStore workerToBuckets; + + + public NamespaceStorage(StoreFactory storageFactory, String pathName) { + super(storageFactory, pathName); + } + + @Override + public void openStores(ObjectMapper objectMapper) { + super.openStores(objectMapper); + + internToExternMappers = getStorageFactory().createInternToExternMappingStore(super.getPathName(), objectMapper); + searchIndexes = getStorageFactory().createSearchIndexStore(super.getPathName(), objectMapper); + idMapping = getStorageFactory().createIdMappingStore(super.getPathName(), objectMapper); + structure = getStorageFactory().createStructureStore(super.getPathName(), objectMapper); + workerToBuckets = getStorageFactory().createWorkerToBucketsStore(super.getPathName(), objectMapper); + preview = getStorageFactory().createPreviewStore(super.getPathName(), objectMapper); + } + + @Override + public ImmutableList getStores() { + return ImmutableList.of( + dataset, + + internToExternMappers, + searchIndexes, + + secondaryIds, + tables, + imports, + + // Concepts depend on internToExternMappers + concepts, + + preview, + idMapping, + structure, + workerToBuckets, + entity2Bucket + ); + } + + + // IdMapping + + public EntityIdMap getIdMapping() { + return idMapping.get(); + } + + public void updateIdMapping(EntityIdMap idMapping) { + this.idMapping.update(idMapping); + } + + // Bucket to Worker Assignment + + public void setWorkerToBucketsMap(WorkerToBucketsMap map) { + workerToBuckets.update(map); + } + + public WorkerToBucketsMap getWorkerBuckets() { + return workerToBuckets.get(); + } + + public int getNumberOfEntities() { + return entity2Bucket.count(); + } + + + public boolean containsEntity(String entity) { + return entity2Bucket.hasKey(entity); + } + + public void registerEntity(String entity, int bucket) { + entity2Bucket.update(entity, bucket); + } + + // Structure + + public StructureNode[] getStructure() { + return Objects.requireNonNullElseGet(structure.get(), () -> new StructureNode[0]); + } + + public void updateStructure(StructureNode[] structure) { + this.structure.update(structure); + } + + // InternToExternMappers + + public InternToExternMapper getInternToExternMapper(InternToExternMapperId id) { + return getInternToExternMapperFromStorage(id); + } + + private InternToExternMapper getInternToExternMapperFromStorage(InternToExternMapperId id) { + return internToExternMappers.get(id); + } + + public void addInternToExternMapper(InternToExternMapper internToExternMapper) { + internToExternMappers.add(internToExternMapper); + } + + public void removeInternToExternMapper(InternToExternMapperId id) { + internToExternMappers.remove(id); + } + + public Stream getInternToExternMappers() { + return internToExternMappers.getAll(); + } + + // SearchIndices + + public SearchIndex getSearchIndex(SearchIndexId id) { + return getSearchIndexFromStorage(id); + } + + private SearchIndex getSearchIndexFromStorage(SearchIndexId id) { + return searchIndexes.get(id); + } + + public void removeSearchIndex(SearchIndexId id) { + searchIndexes.remove(id); + } + + public void addSearchIndex(SearchIndex searchIndex) { + searchIndexes.add(searchIndex); + } + + public Stream getSearchIndices() { + return searchIndexes.getAll(); + } + + // PreviewConfig + + public PreviewConfig getPreviewConfig() { + return preview.get(); + } + + public void setPreviewConfig(PreviewConfig previewConfig) { + preview.update(previewConfig); + } + + public void removePreviewConfig() { + preview.remove(); + } + + // Utilities + + @Override + public MutableInjectableValues inject(MutableInjectableValues values) { + return super.inject(values).add(NamespaceStorage.class, this); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorageImpl.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorageImpl.java index a8c62806955..5309ac3de5d 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorageImpl.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorageImpl.java @@ -74,18 +74,17 @@ public void openStores(ObjectMapper objectMapper) { } - @Override public MutableInjectableValues inject(MutableInjectableValues values) { - return values.add(NamespacedStorage.class, this) - .add(NamespacedStorageProvider.class,this); + return values.add(NamespacedStorage.class, this).add(NamespacedStorageProvider.class, this); } @Override public NamespacedStorage getStorage(@Nullable DatasetId datasetId) { DatasetId thisDatasetId = getDataset().getId(); if (datasetId != null && !datasetId.equals(thisDatasetId)) { - throw new IllegalArgumentException("Dataset id mismatch: expected " + thisDatasetId + " but got " + datasetId); + throw new IllegalArgumentException( + "Dataset id mismatch: expected " + thisDatasetId + " but got " + datasetId); } return this; } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/Store.java b/backend/src/main/java/com/bakdata/conquery/io/storage/Store.java index f5567ef77e6..0e40a1e068f 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/Store.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/Store.java @@ -1,49 +1,49 @@ -package com.bakdata.conquery.io.storage; - -import java.util.stream.Stream; - -import com.bakdata.conquery.io.storage.xodus.stores.SerializingStore.IterationStatistic; - -public interface Store extends ManagedStore { - - /** - * Adds a value to the store, if the key was not present. - * @return True if the value was added - */ - boolean add(KEY key, VALUE value); - - VALUE get(KEY key); - - /** - * Iterate over all key-value pairs in the store. - * @param consumer Key-value consumer. - * @implSpec The consumer must be thread-safe - */ - IterationStatistic forEach(StoreEntryConsumer consumer); - - // TODO: 08.01.2020 fk: Is this still necessary? The implementation in XodusStore uses different methods that in our context don't act differently. - boolean update(KEY key, VALUE value); - - boolean remove(KEY key); - - boolean hasKey(KEY key); - - int count(); - - Stream getAll(); - - Stream getAllKeys(); - - void clear(); - - String getName(); - - /** - * Consumer of key-value pairs stored in this Store. Used in conjunction with for-each. - */ - @FunctionalInterface - interface StoreEntryConsumer { - void accept(KEY key, VALUE value, long size); - } - -} +package com.bakdata.conquery.io.storage; + +import java.util.stream.Stream; + +import com.bakdata.conquery.io.storage.xodus.stores.SerializingStore.IterationStatistic; + +public interface Store extends ManagedStore { + + /** + * Adds a value to the store, if the key was not present. + * @return True if the value was added + */ + boolean add(KEY key, VALUE value); + + VALUE get(KEY key); + + /** + * Iterate over all key-value pairs in the store. + * @param consumer Key-value consumer. + * @implSpec The consumer must be thread-safe + */ + IterationStatistic forEach(StoreEntryConsumer consumer); + + // TODO: 08.01.2020 fk: Is this still necessary? The implementation in XodusStore uses different methods that in our context don't act differently. + boolean update(KEY key, VALUE value); + + boolean remove(KEY key); + + boolean hasKey(KEY key); + + int count(); + + Stream getAll(); + + Stream getAllKeys(); + + void clear(); + + String getName(); + + /** + * Consumer of key-value pairs stored in this Store. Used in conjunction with for-each. + */ + @FunctionalInterface + interface StoreEntryConsumer { + void accept(KEY key, VALUE value, long size); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/StoreMappings.java b/backend/src/main/java/com/bakdata/conquery/io/storage/StoreMappings.java index cd0ce325729..3f6cb6ef420 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/StoreMappings.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/StoreMappings.java @@ -51,27 +51,27 @@ @ToString(of = {"keyType", "valueType"}) public enum StoreMappings { - AUTH_GROUP(Group.class, GroupId.class), - AUTH_ROLE(Role.class, RoleId.class), - AUTH_USER(User.class, UserId.class), - BUCKETS(Bucket.class, BucketId.class), - CONCEPTS(Concept.class, ConceptId.class), - C_BLOCKS(CBlock.class, CBlockId.class), - DATASET(Dataset.class, Boolean.class), - ENTITY_PREVIEW(PreviewConfig.class, Boolean.class), - ENTITY_TO_BUCKET(Integer.class, String.class), - EXECUTIONS(ManagedExecution.class, ManagedExecutionId.class), - FORM_CONFIG(FormConfig.class, FormConfigId.class), - ID_MAPPING(EntityIdMap.class, Boolean.class), - IMPORTS(Import.class, ImportId.class), - INTERN_TO_EXTERN(InternToExternMapper.class, InternToExternMapperId.class), - NAMESPACES(DatasetRegistry.class, Boolean.class), - SEARCH_INDEX(SearchIndex.class, SearchIndexId.class), - SECONDARY_IDS(SecondaryIdDescription.class, SecondaryIdDescriptionId.class), - STRUCTURE(StructureNode[].class, Boolean.class), - TABLES(Table.class, TableId.class), - WORKER(WorkerInformation.class, Boolean.class), - WORKER_TO_BUCKETS(WorkerToBucketsMap.class, Boolean.class); + AUTH_GROUP(Group.class, GroupId.class), AUTH_ROLE(Role.class, RoleId.class), AUTH_USER( + User.class, + UserId.class), BUCKETS(Bucket.class, BucketId.class), CONCEPTS(Concept.class, ConceptId.class), C_BLOCKS( + CBlock.class, + CBlockId.class), DATASET(Dataset.class, Boolean.class), ENTITY_PREVIEW( + PreviewConfig.class, + Boolean.class), ENTITY_TO_BUCKET(Integer.class, String.class), EXECUTIONS( + ManagedExecution.class, + ManagedExecutionId.class), FORM_CONFIG(FormConfig.class, FormConfigId.class), ID_MAPPING( + EntityIdMap.class, + Boolean.class), IMPORTS(Import.class, ImportId.class), INTERN_TO_EXTERN( + InternToExternMapper.class, + InternToExternMapperId.class), NAMESPACES( + DatasetRegistry.class, + Boolean.class), SEARCH_INDEX(SearchIndex.class, SearchIndexId.class), SECONDARY_IDS( + SecondaryIdDescription.class, + SecondaryIdDescriptionId.class), STRUCTURE( + StructureNode[].class, + Boolean.class), TABLES(Table.class, TableId.class), WORKER( + WorkerInformation.class, + Boolean.class), WORKER_TO_BUCKETS(WorkerToBucketsMap.class, Boolean.class); private final Class valueType; private final Class keyType; diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/WorkerStorageImpl.java b/backend/src/main/java/com/bakdata/conquery/io/storage/WorkerStorageImpl.java index 8e8fc516da5..729a6bea915 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/WorkerStorageImpl.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/WorkerStorageImpl.java @@ -30,16 +30,16 @@ public WorkerStorageImpl(StoreFactory storageFactory, String pathName) { @Override public ImmutableList getStores() { return ImmutableList.of( - dataset, - secondaryIds, - tables, - imports, - concepts, - - entity2Bucket, - worker, - buckets, - cBlocks + dataset, + secondaryIds, + tables, + imports, + concepts, + + entity2Bucket, + worker, + buckets, + cBlocks ); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/BigStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/BigStore.java index 9f9959acbe1..3b278dac095 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/BigStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/BigStore.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.io.storage.xodus.stores; +import jakarta.validation.Validator; +import jakarta.validation.constraints.NotEmpty; import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; import java.io.Closeable; @@ -15,8 +17,6 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; import java.util.stream.Stream; -import jakarta.validation.Validator; -import jakarta.validation.constraints.NotEmpty; import com.bakdata.conquery.io.storage.Store; import com.bakdata.conquery.io.storage.xodus.stores.SerializingStore.IterationStatistic; @@ -48,17 +48,20 @@ public class BigStore implements Store, Closeable { private final XodusStore dataXodusStore; private final StoreInfo storeInfo; private final ObjectReader valueReader; - @Getter @Setter + @Getter + @Setter private int chunkByteSize; - public BigStore(XodusStoreFactory config, - Validator validator, - Environment env, - StoreInfo storeInfo, - Consumer storeCloseHook, - Consumer storeRemoveHook, - ObjectMapper mapper, ExecutorService executorService) { + public BigStore( + XodusStoreFactory config, + Validator validator, + Environment env, + StoreInfo storeInfo, + Consumer storeCloseHook, + Consumer storeRemoveHook, + ObjectMapper mapper, + ExecutorService executorService) { this.storeInfo = storeInfo; // Recommendation by the author of Xodus is to have logFileSize at least be 4 times the biggest file size. @@ -66,31 +69,32 @@ public BigStore(XodusStoreFactory config, metaXodusStore = new XodusStore(env, storeInfo.getName() + "_META", storeCloseHook, storeRemoveHook); metaStore = new SerializingStore<>( - metaXodusStore, - validator, - mapper, - storeInfo.getKeyType(), - BigStoreMetaKeys.class, - config.isValidateOnWrite(), - config.isRemoveUnreadableFromStore(), - config.getUnreadableDataDumpDirectory(), executorService + metaXodusStore, + validator, + mapper, + storeInfo.getKeyType(), + BigStoreMetaKeys.class, + config.isValidateOnWrite(), + config.isRemoveUnreadableFromStore(), + config.getUnreadableDataDumpDirectory(), + executorService ); dataXodusStore = new XodusStore(env, storeInfo.getName() + "_DATA", storeCloseHook, storeRemoveHook); dataStore = new SerializingStore<>( - dataXodusStore, - validator, - mapper, - UUID.class, - byte[].class, - config.isValidateOnWrite(), - config.isRemoveUnreadableFromStore(), - config.getUnreadableDataDumpDirectory(), executorService + dataXodusStore, + validator, + mapper, + UUID.class, + byte[].class, + config.isValidateOnWrite(), + config.isRemoveUnreadableFromStore(), + config.getUnreadableDataDumpDirectory(), + executorService ); - this.valueWriter = mapper.writerFor(storeInfo.getValueType()); this.valueReader = mapper.readerFor(storeInfo.getValueType()); } @@ -105,9 +109,7 @@ public VALUE get(KEY key) { } private VALUE createValue(KEY key, BigStoreMetaKeys meta) { - Iterator it = meta.loadData(dataStore) - .map(ByteArrayInputStream::new) - .iterator(); + Iterator it = meta.loadData(dataStore).map(ByteArrayInputStream::new).iterator(); try (InputStream in = new BufferedInputStream(new SequenceInputStream(IteratorUtils.asEnumeration(it)))) { return valueReader.readValue(in); @@ -161,25 +163,24 @@ private BigStoreMetaKeys writeValue(VALUE value) { List uuids = new ArrayList<>(); ChunkingOutputStream cos = new ChunkingOutputStream( - chunkByteSize, - chunk -> { - try { - // Write chunks and accumulate their size. - UUID id = UUID.randomUUID(); - uuids.add(id); - dataStore.add(id, chunk); - size.addAndGet(chunk.length); - } - catch (Exception e) { - throw new RuntimeException("Failed to write chunk", e); - } + chunkByteSize, + chunk -> { + try { + // Write chunks and accumulate their size. + UUID id = UUID.randomUUID(); + uuids.add(id); + dataStore.add(id, chunk); + size.addAndGet(chunk.length); + } catch (Exception e) { + throw new RuntimeException("Failed to write chunk", e); } + } ); try (cos) { valueWriter.writeValue( - cos, - value + cos, + value ); } return new BigStoreMetaKeys(uuids.toArray(new UUID[0]), size.get()); diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/CachedStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/CachedStore.java index 1389f2b473e..42b24eae178 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/CachedStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/CachedStore.java @@ -45,14 +45,15 @@ public class CachedStore implements Store { public CachedStore(Store store, CaffeineSpec caffeineSpec, MetricRegistry metricRegistry) { this.store = store; - final StatsCounter statsCounter = - metricRegistry != null ? new MetricsStatsCounter(metricRegistry, "cache." + store.toString()) : StatsCounter.disabledStatsCounter(); + final StatsCounter statsCounter = metricRegistry != null ? new MetricsStatsCounter( + metricRegistry, + "cache." + store.toString()) : StatsCounter.disabledStatsCounter(); final Caffeine caffeine = Caffeine.from(caffeineSpec) - .recordStats(() -> statsCounter) - .evictionListener( - (k, v, cause) -> - log.trace("Evicting {} from cache for {} (cause: {})", k, store.toString(), cause)); + .recordStats( + () -> statsCounter) + .evictionListener( + (k, v, cause) -> log.trace("Evicting {} from cache for {} (cause: {})", k, store.toString(), cause)); cache = caffeine.build(this::getFromStore); } @@ -143,8 +144,7 @@ public void loadData() { if (count < 100) { bar = null; - } - else { + } else { bar = PROGRESS_BAR; bar.addMaxValue(count); } @@ -164,22 +164,27 @@ public void loadData() { if (!dupes.add(key)) { log.warn("Multiple Keys deserialize to `{}`", key); } - } - catch (RuntimeException e) { + } catch (RuntimeException e) { if (e.getCause() != null && e.getCause() instanceof IdReferenceResolvingException) { - log.warn("Probably failed to read id '{}' because it is not yet present, skipping", ((IdReferenceResolvingException) e.getCause()).getValue(), e); - } - else { + log.warn( + "Probably failed to read id '{}' because it is not yet present, skipping", + ((IdReferenceResolvingException) e.getCause()).getValue(), + e); + } else { throw e; } - } - finally { + } finally { if (bar != null) { bar.addCurrentValue(1); } } }); - log.debug("\tloaded store {}: {} entries, {} within {}", this, count, FileUtils.byteCountToDisplaySize(totalSize.sum()), timer.stop()); + log.debug( + "\tloaded store {}: {} entries, {} within {}", + this, + count, + FileUtils.byteCountToDisplaySize(totalSize.sum()), + timer.stop()); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/EnvironmentRegistry.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/EnvironmentRegistry.java index d91e1ca3768..36638029cc8 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/EnvironmentRegistry.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/EnvironmentRegistry.java @@ -46,7 +46,9 @@ private void unregister(Environment environment) { final Environment remove = activeEnvironments.remove(environment.getLocation()); if (remove == null) { - log.warn("Could not unregister environment, because it was not registered: {}", environment.getLocation()); + log.warn( + "Could not unregister environment, because it was not registered: {}", + environment.getLocation()); } } } @@ -57,11 +59,10 @@ public Environment findOrCreateEnvironment(@NonNull File path, XodusConfig xodus try { // Check for old env or register new env return activeEnvironments.computeIfAbsent( - path.toString(), - newPath -> createManaged(Environments.newInstance(newPath, xodusConfig.createConfig())) + path.toString(), + newPath -> createManaged(Environments.newInstance(newPath, xodusConfig.createConfig())) ); - } - catch (Exception e) { + } catch (Exception e) { throw new IllegalStateException("Unable to open environment: " + path, e); } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/KeyIncludingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/KeyIncludingStore.java index e6bac4b69c6..e9a94802031 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/KeyIncludingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/KeyIncludingStore.java @@ -1,57 +1,54 @@ -package com.bakdata.conquery.io.storage.xodus.stores; - -import java.io.Closeable; -import java.util.Objects; -import java.util.stream.Stream; - -import com.bakdata.conquery.io.storage.ManagedStore; -import com.bakdata.conquery.io.storage.Store; -import lombok.Data; -import lombok.experimental.Delegate; -import lombok.extern.slf4j.Slf4j; - -@Data -@Slf4j -public abstract class KeyIncludingStore implements Closeable, ManagedStore { - - @Delegate - protected final Store store; - - public void add(VALUE value) { - store.add(extractKey(value), value); - } - - protected abstract KEY extractKey(VALUE value); - - public boolean hasKey(KEY key) { - return store.hasKey(key); - } - - public void update(VALUE value) { - store.update(extractKey(value), value); - } - - public VALUE get(KEY key) { - return store.get(key); - } - - public Stream getAll() { - return store.getAllKeys() - .map(this::getIgnoringExceptions) - .filter(Objects::nonNull); - } - - /** - * Gets the value for a key if it is present and can be loaded. - * If the value could not be loaded, returns null. - */ - private VALUE getIgnoringExceptions(KEY key) { - try { - return get(key); - } - catch (Exception e) { - log.trace("Unable to load value for key {}", key, e); - return null; - } - } -} +package com.bakdata.conquery.io.storage.xodus.stores; + +import java.io.Closeable; +import java.util.Objects; +import java.util.stream.Stream; + +import com.bakdata.conquery.io.storage.ManagedStore; +import com.bakdata.conquery.io.storage.Store; +import lombok.Data; +import lombok.experimental.Delegate; +import lombok.extern.slf4j.Slf4j; + +@Data +@Slf4j +public abstract class KeyIncludingStore implements Closeable, ManagedStore { + + @Delegate + protected final Store store; + + public void add(VALUE value) { + store.add(extractKey(value), value); + } + + protected abstract KEY extractKey(VALUE value); + + public boolean hasKey(KEY key) { + return store.hasKey(key); + } + + public void update(VALUE value) { + store.update(extractKey(value), value); + } + + public VALUE get(KEY key) { + return store.get(key); + } + + public Stream getAll() { + return store.getAllKeys().map(this::getIgnoringExceptions).filter(Objects::nonNull); + } + + /** + * Gets the value for a key if it is present and can be loaded. + * If the value could not be loaded, returns null. + */ + private VALUE getIgnoringExceptions(KEY key) { + try { + return get(key); + } catch (Exception e) { + log.trace("Unable to load value for key {}", key, e); + return null; + } + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index 75d1b320fa0..ec7420c0e06 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.io.storage.xodus.stores; +import jakarta.validation.Validator; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; @@ -27,7 +28,6 @@ import java.util.stream.Stream; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; -import jakarta.validation.Validator; import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.jackson.JacksonUtil; @@ -122,15 +122,15 @@ public class SerializingStore implements Store { private final ExecutorService executor; public , CLASS_V extends Class> SerializingStore( - Store store, - Validator validator, - ObjectMapper objectMapper, - CLASS_K keyType, - CLASS_V valueType, - boolean validateOnWrite, - boolean removeUnreadableFromStore, - File unreadableDataDumpDirectory, - ExecutorService executorService) { + Store store, + Validator validator, + ObjectMapper objectMapper, + CLASS_K keyType, + CLASS_V valueType, + boolean validateOnWrite, + boolean removeUnreadableFromStore, + File unreadableDataDumpDirectory, + ExecutorService executorService) { this.store = store; this.validator = validator; this.validateOnWrite = validateOnWrite; @@ -155,10 +155,13 @@ public , CLASS_V extends Class> SerializingSto if (shouldDumpUnreadables()) { if (!unreadableValuesDumpDir.exists() && !unreadableValuesDumpDir.mkdirs()) { - throw new IllegalStateException("Could not create dump directory: %s".formatted(unreadableValuesDumpDir)); - } - else if (!unreadableValuesDumpDir.isDirectory()) { - throw new IllegalArgumentException(String.format("The provided path points to an existing file which is not a directory. Was: %s", unreadableValuesDumpDir.getAbsolutePath())); + throw new IllegalStateException( + "Could not create dump directory: %s".formatted(unreadableValuesDumpDir)); + } else if (!unreadableValuesDumpDir.isDirectory()) { + throw new IllegalArgumentException( + String.format( + "The provided path points to an existing file which is not a directory. Was: %s", + unreadableValuesDumpDir.getAbsolutePath())); } } } @@ -170,7 +173,8 @@ private boolean shouldDumpUnreadables() { @Override public boolean add(KEY key, VALUE value) { if (!valueType.isInstance(value)) { - throw new IllegalStateException("The element %s is not of the required type %s".formatted(value, valueType)); + throw new IllegalStateException( + "The element %s is not of the required type %s".formatted(value, valueType)); } if (validateOnWrite) { ValidatorHelper.failOnError(log, validator.validate(value)); @@ -209,8 +213,7 @@ private ByteIterable write(Object obj, ObjectWriter writer) { final byte[] bytes = baos.toByteArray(); return new ArrayByteIterable(bytes); - } - catch (IOException e) { + } catch (IOException e) { throw new RuntimeException("Failed to write " + obj, e); } } @@ -221,11 +224,16 @@ public VALUE get(KEY key) { try { return readValue(binValue); - } - catch (Exception e) { + } catch (Exception e) { if (unreadableValuesDumpDir != null) { - dumpToFile(binValue.getBytesUnsafe(), key.toString(), e, unreadableValuesDumpDir, getName(), objectMapper); + dumpToFile( + binValue.getBytesUnsafe(), + key.toString(), + e, + unreadableValuesDumpDir, + getName(), + objectMapper); } if (removeUnreadablesFromUnderlyingStore) { @@ -256,7 +264,13 @@ private VALUE readValue(ByteIterable value) { * @param unreadableDumpDir The director to dump to. The method assumes that the directory exists and is okay to write to. * @param storeName The name of the store which is also used in the dump file name. */ - private static void dumpToFile(byte[] gzippedObj, @NonNull String keyOfDump, Exception reason, @NonNull File unreadableDumpDir, String storeName, ObjectMapper objectMapper) { + private static void dumpToFile( + byte[] gzippedObj, + @NonNull String keyOfDump, + Exception reason, + @NonNull File unreadableDumpDir, + String storeName, + ObjectMapper objectMapper) { // Create dump filehandle final File dumpfile = makeDumpFileName(keyOfDump, unreadableDumpDir, storeName); final File exceptionFileName = makeExceptionFileName(keyOfDump, unreadableDumpDir, storeName); @@ -274,26 +288,28 @@ private static void dumpToFile(byte[] gzippedObj, @NonNull String keyOfDump, Exc if (!dumpfile.getParentFile().exists()) { throw new IllegalStateException("Could not create `%s`.".formatted(dumpfile.getParentFile())); } - } - catch (IOException e) { + } catch (IOException e) { log.warn("Could not create `{}`", dumpfile.getParentFile(), e); } // Write json try { - log.info("Dumping value of key {} to {} (because it cannot be deserialized anymore).", keyOfDump, dumpfile.getCanonicalPath()); - - final JsonNode dump = objectMapper.readerFor(JsonNode.class).readValue(new GZIPInputStream(new ByteArrayInputStream(gzippedObj))); + log.info( + "Dumping value of key {} to {} (because it cannot be deserialized anymore).", + keyOfDump, + dumpfile.getCanonicalPath()); + + final JsonNode dump = objectMapper.readerFor(JsonNode.class) + .readValue( + new GZIPInputStream(new ByteArrayInputStream(gzippedObj))); Jackson.MAPPER.writer().writeValue(dumpfile, dump); - } - catch (IOException e) { + } catch (IOException e) { log.error("Failed to dump unreadable value of key `{}` to file `{}`", keyOfDump, dumpfile, e); } try (PrintStream out = new PrintStream(exceptionFileName)) { reason.printStackTrace(out); - } - catch (IOException e) { + } catch (IOException e) { log.error("Failed to dump exception for `{}` to file `{}`.", keyOfDump, exceptionFileName, e); } @@ -317,14 +333,15 @@ private T read(ObjectReader reader, ByteIterable obj) { if (obj == null) { return null; } - try (final InputStream inputStream = new GZIPInputStream(new ByteArrayInputStream(obj.getBytesUnsafe(), 0, obj.getLength()))) { + try (final InputStream inputStream = new GZIPInputStream( + new ByteArrayInputStream(obj.getBytesUnsafe(), 0, obj.getLength()))) { return reader.readValue(inputStream); - } - catch (IOException e) { + } catch (IOException e) { try { - throw new RuntimeException("Failed to read " + JacksonUtil.toJsonDebug(debugUnGzip(obj.getBytesUnsafe())), e); - } - catch (IOException ex) { + throw new RuntimeException( + "Failed to read " + JacksonUtil.toJsonDebug(debugUnGzip(obj.getBytesUnsafe())), + e); + } catch (IOException ex) { throw new RuntimeException(ex); } } @@ -337,12 +354,16 @@ private T read(ObjectReader reader, ByteIterable obj) { * Current implementation is `$unreadableDumpDir/$today/$store/$key.json` */ @NotNull - public static File makeDumpFileName(@NotNull String keyOfDump, @NotNull File unreadableDumpDir, @NotNull String storeName) { + public static File makeDumpFileName( + @NotNull String keyOfDump, + @NotNull File unreadableDumpDir, + @NotNull String storeName) { return unreadableDumpDir.toPath() - .resolve(DateTimeFormatter.BASIC_ISO_DATE.format(LocalDateTime.now())) - .resolve(storeName) - .resolve(sanitiseFileName(keyOfDump) + "." + DUMP_FILE_EXTENSION) - .toFile(); + .resolve(DateTimeFormatter.BASIC_ISO_DATE.format(LocalDateTime.now())) + .resolve( + storeName) + .resolve(sanitiseFileName(keyOfDump) + "." + DUMP_FILE_EXTENSION) + .toFile(); } @@ -353,12 +374,16 @@ public static File makeDumpFileName(@NotNull String keyOfDump, @NotNull File unr * Current implementation is `$unreadableDumpDir/$today/$store/$key.exception` */ @NotNull - public static File makeExceptionFileName(@NotNull String keyOfDump, @NotNull File unreadableDumpDir, @NotNull String storeName) { + public static File makeExceptionFileName( + @NotNull String keyOfDump, + @NotNull File unreadableDumpDir, + @NotNull String storeName) { return unreadableDumpDir.toPath() - .resolve(DateTimeFormatter.BASIC_ISO_DATE.format(LocalDateTime.now())) - .resolve(storeName) - .resolve(sanitiseFileName(keyOfDump) + "." + EXCEPTION_FILE_EXTENSION) - .toFile(); + .resolve(DateTimeFormatter.BASIC_ISO_DATE.format(LocalDateTime.now())) + .resolve( + storeName) + .resolve(sanitiseFileName(keyOfDump) + "." + EXCEPTION_FILE_EXTENSION) + .toFile(); } @@ -396,20 +421,17 @@ public IterationStatistic forEach(StoreEntryConsumer consumer) { do { try { maybeFailed = allJobs.get(30, TimeUnit.SECONDS); - } - catch (InterruptedException e) { + } catch (InterruptedException e) { boolean interrupted = Thread.currentThread().isInterrupted(); log.debug("Waiting for deserialization was interrupted: {}", interrupted); - if(interrupted) { + if (interrupted) { // Rethrow to propagate Thread.currentThread().interrupt(); } - } - catch (ExecutionException e) { + } catch (ExecutionException e) { throw new RuntimeException(e); - } - catch (TimeoutException e) { + } catch (TimeoutException e) { log.debug("Still waiting for {} jobs.", jobs.stream().filter(Predicate.not(Future::isDone)).count()); } } while (!allJobs.isDone()); @@ -420,7 +442,8 @@ public IterationStatistic forEach(StoreEntryConsumer consumer) { // Print some statistics final int total = result.getTotalProcessed(); - log.debug(String.format( + log.debug( + String.format( "While processing store %s:\n\tEntries processed:\t%d\n\tKey read failure:\t%d (%.2f%%)\n\tValue read failure:\t%d (%.2f%%)", store.getName(), total, @@ -428,7 +451,7 @@ public IterationStatistic forEach(StoreEntryConsumer consumer) { total > 0 ? (float) result.getFailedKeys() / total * 100 : 0, result.getFailedValues(), total > 0 ? (float) result.getFailedValues() / total * 100 : 0 - )); + )); // Remove corrupted entries from the store if configured so if (removeUnreadablesFromUnderlyingStore) { @@ -438,41 +461,51 @@ public IterationStatistic forEach(StoreEntryConsumer consumer) { return result; } - private ByteIterable handle(StoreEntryConsumer consumer, IterationStatistic result, ByteIterable keyRaw, ByteIterable valueRaw) { + private ByteIterable handle( + StoreEntryConsumer consumer, + IterationStatistic result, + ByteIterable keyRaw, + ByteIterable valueRaw) { final KEY key; final VALUE value; try { // Try to read the key first - key = - getDeserializedAndDumpFailed(keyRaw, SerializingStore.this::readKey, () -> new String(keyRaw.getBytesUnsafe()), valueRaw, "Could not parse key [{}]"); + key = getDeserializedAndDumpFailed( + keyRaw, + SerializingStore.this::readKey, + () -> new String(keyRaw.getBytesUnsafe()), + valueRaw, + "Could not parse key [{}]"); if (key == null) { result.incrFailedKeys(); return keyRaw; } // Try to read the value - value = getDeserializedAndDumpFailed(valueRaw, SerializingStore.this::readValue, key::toString, valueRaw, "Could not parse value for key [{}]"); + value = getDeserializedAndDumpFailed( + valueRaw, + SerializingStore.this::readValue, + key::toString, + valueRaw, + "Could not parse value for key [{}]"); if (value == null) { result.incrFailedValues(); return keyRaw; } - } - catch (Exception e) { + } catch (Exception e) { log.error("Failed processing key/value", e); return keyRaw; - } - finally { + } finally { result.incrTotalProcessed(); } // Apply the consumer to key and value try { consumer.accept(key, value, valueRaw.getLength()); - } - catch (Exception e) { + } catch (Exception e) { log.warn("Unable to apply for-each consumer on key[{}]", key, e); } @@ -490,16 +523,26 @@ private ByteIterable handle(StoreEntryConsumer consumer, IterationSt * @param onFailWarnMsgFmt The warning message that will be logged on failure. * @return The deserialized value */ - private TYPE getDeserializedAndDumpFailed(ByteIterable serial, Function deserializer, Supplier onFailKeyStringSupplier, ByteIterable onFailOrigValue, String onFailWarnMsgFmt) { + private TYPE getDeserializedAndDumpFailed( + ByteIterable serial, + Function deserializer, + Supplier onFailKeyStringSupplier, + ByteIterable onFailOrigValue, + String onFailWarnMsgFmt) { try { return deserializer.apply(serial); - } - catch (Exception e) { + } catch (Exception e) { // With trace also print the stacktrace log.warn(onFailWarnMsgFmt, onFailKeyStringSupplier.get(), log.isTraceEnabled() ? e : null); if (shouldDumpUnreadables()) { - dumpToFile(onFailOrigValue.getBytesUnsafe(), onFailKeyStringSupplier.get(), e, unreadableValuesDumpDir, store.getName(), objectMapper); + dumpToFile( + onFailOrigValue.getBytesUnsafe(), + onFailKeyStringSupplier.get(), + e, + unreadableValuesDumpDir, + store.getName(), + objectMapper); } } return null; @@ -508,7 +551,8 @@ private TYPE getDeserializedAndDumpFailed(ByteIterable serial, Function { +@Data +@AllArgsConstructor +public class StoreInfo { private final String name; private final Class keyType; diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/XodusStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/XodusStore.java index 586dbde9d65..6c148eb30ea 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/XodusStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/XodusStore.java @@ -29,7 +29,11 @@ public class XodusStore implements com.bakdata.conquery.io.storage.Store storeCloseHook, Consumer storeRemoveHook) { + public XodusStore( + Environment env, + String name, + Consumer storeCloseHook, + Consumer storeRemoveHook) { // Arbitrary duration that is strictly shorter than the timeout to not get interrupted by StuckTxMonitor this.timeoutHalfMillis = env.getEnvironmentConfig().getEnvMonitorTxnsTimeout() / 2; this.name = name; @@ -37,7 +41,7 @@ public XodusStore(Environment env, String name, Consumer storeCloseH this.storeCloseHook = storeCloseHook; this.storeRemoveHook = storeRemoveHook; this.store = env.computeInTransaction( - t -> env.openStore(this.name, StoreConfig.WITHOUT_DUPLICATES_WITH_PREFIXING, t) + t -> env.openStore(this.name, StoreConfig.WITHOUT_DUPLICATES_WITH_PREFIXING, t) ); } @@ -208,8 +212,7 @@ public boolean tryAdvance(Consumer action) { lastKey.set(key); action.accept(key); return true; - } - else { + } else { cursor.close(); onClose(); return false; diff --git a/backend/src/main/java/com/bakdata/conquery/metrics/ActiveUsersFilter.java b/backend/src/main/java/com/bakdata/conquery/metrics/ActiveUsersFilter.java index 9b84d6619d2..5f97dea18e7 100644 --- a/backend/src/main/java/com/bakdata/conquery/metrics/ActiveUsersFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/metrics/ActiveUsersFilter.java @@ -1,5 +1,11 @@ package com.bakdata.conquery.metrics; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.Priority; +import jakarta.inject.Inject; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.container.ContainerRequestFilter; +import jakarta.ws.rs.container.PreMatching; import java.security.Principal; import java.time.Duration; import java.time.LocalDateTime; @@ -16,12 +22,6 @@ import com.codahale.metrics.SharedMetricRegistries; import com.google.common.collect.HashBasedTable; import com.google.common.collect.Table; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.Priority; -import jakarta.inject.Inject; -import jakarta.ws.rs.container.ContainerRequestContext; -import jakarta.ws.rs.container.ContainerRequestFilter; -import jakarta.ws.rs.container.PreMatching; import lombok.extern.slf4j.Slf4j; /** @@ -80,11 +80,11 @@ public void filter(ContainerRequestContext requestContext) { /** * Count the number of users who have issued a request in the configured duration. */ - private Gauge activeUsersGauge(Group group){ + private Gauge activeUsersGauge(Group group) { return () -> { int active = 0; for (Map.Entry usageTimes : activeUsers.row(group).entrySet()) { - if(usageTimes.getValue().isBefore(LocalDateTime.now().minus(activeUserDuration))){ + if (usageTimes.getValue().isBefore(LocalDateTime.now().minus(activeUserDuration))) { continue; } diff --git a/backend/src/main/java/com/bakdata/conquery/metrics/ExecutionMetrics.java b/backend/src/main/java/com/bakdata/conquery/metrics/ExecutionMetrics.java index 775cdda3910..7d379914399 100644 --- a/backend/src/main/java/com/bakdata/conquery/metrics/ExecutionMetrics.java +++ b/backend/src/main/java/com/bakdata/conquery/metrics/ExecutionMetrics.java @@ -42,7 +42,9 @@ public class ExecutionMetrics { private static final String TIME = "time"; public static Counter getRunningQueriesCounter(String group) { - return SharedMetricRegistries.getDefault().counter(nameWithGroupTag(MetricRegistry.name(QUERIES, RUNNING), group)); + return SharedMetricRegistries.getDefault() + .counter( + nameWithGroupTag(MetricRegistry.name(QUERIES, RUNNING), group)); } /** @@ -65,13 +67,16 @@ public static Histogram getQueriesTimeHistogram(String group) { } public static Counter getQueryStateCounter(ExecutionState state, String group) { - return SharedMetricRegistries.getDefault().counter(nameWithGroupTag(MetricRegistry.name(QUERIES, STATE, state.toString()), group)); + return SharedMetricRegistries.getDefault() + .counter( + nameWithGroupTag(MetricRegistry.name(QUERIES, STATE, state.toString()), group)); } public static void reportQueryClassUsage(Class clazz, String group) { SharedMetricRegistries.getDefault() - .counter(nameWithGroupTag(MetricRegistry.name(QUERIES, CLASSES, clazz.getSimpleName()), group)) - .inc(); // Count usages of different types of Queries + .counter( + nameWithGroupTag(MetricRegistry.name(QUERIES, CLASSES, clazz.getSimpleName()), group)) + .inc(); // Count usages of different types of Queries } /** @@ -100,7 +105,10 @@ public static void reportNamespacedIds(Collection> fou } for (ConceptId id : reportedIds) { - SharedMetricRegistries.getDefault().counter(nameWithGroupTag(MetricRegistry.name(QUERIES, CONCEPTS, id.toString()), group)).inc(); + SharedMetricRegistries.getDefault() + .counter( + nameWithGroupTag(MetricRegistry.name(QUERIES, CONCEPTS, id.toString()), group)) + .inc(); } } @@ -154,9 +162,7 @@ public void doReport(String category, String id) { return; } - SharedMetricRegistries.getDefault() - .counter(name) - .inc(); + SharedMetricRegistries.getDefault().counter(name).inc(); } } } diff --git a/backend/src/main/java/com/bakdata/conquery/metrics/JobMetrics.java b/backend/src/main/java/com/bakdata/conquery/metrics/JobMetrics.java index 82d74f4e3a3..75dff466e81 100644 --- a/backend/src/main/java/com/bakdata/conquery/metrics/JobMetrics.java +++ b/backend/src/main/java/com/bakdata/conquery/metrics/JobMetrics.java @@ -1,34 +1,40 @@ -package com.bakdata.conquery.metrics; - -import java.util.concurrent.LinkedBlockingDeque; - -import com.bakdata.conquery.io.storage.ConqueryStorage; -import com.bakdata.conquery.models.jobs.Job; -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.SharedMetricRegistries; -import com.codahale.metrics.Timer; -import lombok.experimental.UtilityClass; - -@UtilityClass -public class JobMetrics { - - private static final String JOBS = "jobs"; - private static final String EXECUTION_TIME = "time"; - private static final String QUEUE_SIZE = "queue"; - - public static Timer.Context getStoreLoadingTimer() { - return SharedMetricRegistries.getDefault().timer(MetricRegistry.name(JOBS, EXECUTION_TIME, ConqueryStorage.class.getSimpleName())).time(); - } - - public static void createJobQueueGauge(String name, LinkedBlockingDeque jobs) { - SharedMetricRegistries.getDefault().gauge(MetricRegistry.name(JOBS, QUEUE_SIZE, name), () -> jobs::size); - } - - public static void removeJobQueueSizeGauge(String name) { - SharedMetricRegistries.getDefault().remove(MetricRegistry.name(JOBS, QUEUE_SIZE, name)); - } - - public static Timer.Context getJobExecutorTimer(Job job) { - return SharedMetricRegistries.getDefault().timer(MetricRegistry.name(JOBS, EXECUTION_TIME, job.getClass().getSimpleName())).time(); - } -} +package com.bakdata.conquery.metrics; + +import java.util.concurrent.LinkedBlockingDeque; + +import com.bakdata.conquery.io.storage.ConqueryStorage; +import com.bakdata.conquery.models.jobs.Job; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.SharedMetricRegistries; +import com.codahale.metrics.Timer; +import lombok.experimental.UtilityClass; + +@UtilityClass +public class JobMetrics { + + private static final String JOBS = "jobs"; + private static final String EXECUTION_TIME = "time"; + private static final String QUEUE_SIZE = "queue"; + + public static Timer.Context getStoreLoadingTimer() { + return SharedMetricRegistries.getDefault() + .timer( + MetricRegistry.name(JOBS, EXECUTION_TIME, ConqueryStorage.class.getSimpleName())) + .time(); + } + + public static void createJobQueueGauge(String name, LinkedBlockingDeque jobs) { + SharedMetricRegistries.getDefault().gauge(MetricRegistry.name(JOBS, QUEUE_SIZE, name), () -> jobs::size); + } + + public static void removeJobQueueSizeGauge(String name) { + SharedMetricRegistries.getDefault().remove(MetricRegistry.name(JOBS, QUEUE_SIZE, name)); + } + + public static Timer.Context getJobExecutorTimer(Job job) { + return SharedMetricRegistries.getDefault() + .timer( + MetricRegistry.name(JOBS, EXECUTION_TIME, job.getClass().getSimpleName())) + .time(); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/metrics/prometheus/PrometheusBundle.java b/backend/src/main/java/com/bakdata/conquery/metrics/prometheus/PrometheusBundle.java index 019aa7e556f..28acfb13115 100644 --- a/backend/src/main/java/com/bakdata/conquery/metrics/prometheus/PrometheusBundle.java +++ b/backend/src/main/java/com/bakdata/conquery/metrics/prometheus/PrometheusBundle.java @@ -23,6 +23,9 @@ public void initialize(Bootstrap bootstrap) { @Override public void run(ConqueryConfig configuration, Environment environment) throws Exception { - environment.getAdminContext().addServlet(new ServletHolder(new PrometheusMetricsServlet()), "/metrics-prometheus"); + environment.getAdminContext() + .addServlet( + new ServletHolder(new PrometheusMetricsServlet()), + "/metrics-prometheus"); } } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/Manager.java b/backend/src/main/java/com/bakdata/conquery/mode/Manager.java index 6d538f99b1a..162f3385ada 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/Manager.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/Manager.java @@ -20,13 +20,21 @@ */ public interface Manager extends Managed { ConqueryConfig getConfig(); + Environment getEnvironment(); + DatasetRegistry getDatasetRegistry(); + ImportHandler getImportHandler(); + StorageListener getStorageListener(); + Supplier> getNodeProvider(); + List getAdminTasks(); + InternalMapperFactory getInternalMapperFactory(); + JobManager getJobManager(); MetaStorage getMetaStorage(); diff --git a/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java index bb96f072b8e..44b3fc81da7 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java @@ -22,16 +22,16 @@ static JobManager newJobManager(ConqueryConfig config) { } static DatasetRegistry createDatasetRegistry( - NamespaceHandler namespaceHandler, - ConqueryConfig config, - InternalMapperFactory internalMapperFactory + NamespaceHandler namespaceHandler, + ConqueryConfig config, + InternalMapperFactory internalMapperFactory ) { final IndexService indexService = new IndexService(config.getCsv().createCsvParserSettings()); return new DatasetRegistry<>( - config, - internalMapperFactory, - namespaceHandler, - indexService + config, + internalMapperFactory, + namespaceHandler, + indexService ); } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java index 3ef47152729..77f806264c8 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java @@ -28,12 +28,19 @@ public interface NamespaceHandler { /** * Creates the {@link NamespaceSetupData} that is shared by all {@link Namespace} types. */ - static NamespaceSetupData createNamespaceSetup(NamespaceStorage storage, final ConqueryConfig config, final InternalMapperFactory internalMapperFactory, DatasetRegistry datasetRegistry, Environment environment) { + static NamespaceSetupData createNamespaceSetup( + NamespaceStorage storage, + final ConqueryConfig config, + final InternalMapperFactory internalMapperFactory, + DatasetRegistry datasetRegistry, + Environment environment) { List injectables = new ArrayList<>(); injectables.add(datasetRegistry); injectables.add(storage); - ObjectMapper persistenceMapper = internalMapperFactory.createNamespacePersistenceMapper(storage, datasetRegistry); + ObjectMapper persistenceMapper = internalMapperFactory.createNamespacePersistenceMapper( + storage, + datasetRegistry); ObjectMapper preprocessMapper = internalMapperFactory.createPreprocessMapper(storage, datasetRegistry); injectables.forEach(i -> { @@ -58,7 +65,11 @@ static NamespaceSetupData createNamespaceSetup(NamespaceStorage storage, final C return new NamespaceSetupData(preprocessMapper, jobManager, filterSearch); } - N createNamespace(NamespaceStorage namespaceStorage, MetaStorage metaStorage, DatasetRegistry datasetRegistry, Environment environment); + N createNamespace( + NamespaceStorage namespaceStorage, + MetaStorage metaStorage, + DatasetRegistry datasetRegistry, + Environment environment); void removeNamespace(DatasetId id, N namespace); diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionManager.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionManager.java index f87ad8a0638..9722267d979 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionManager.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionManager.java @@ -70,28 +70,28 @@ public void messageReceived(IoSession session, Object message) { if (shardNodeInformation == null) { // In case the shard is not yet registered, we won't have a shardNodeInformation to pull the session from nwSession = new NetworkSession(session, config.getCluster().getNetworkSessionMaxQueueLength()); - } - else { + } else { nwSession = shardNodeInformation.getSession(); } log.trace("ManagerNode received {} from {}", message.getClass().getSimpleName(), session.getRemoteAddress()); - final Job job = new ReactingJob<>(toManagerNode, new NetworkMessageContext.ManagerNodeNetworkContext(nwSession, datasetRegistry, clusterState)); + final Job job = new ReactingJob<>( + toManagerNode, + new NetworkMessageContext.ManagerNodeNetworkContext(nwSession, datasetRegistry, clusterState)); if (toManagerNode instanceof ForwardToNamespace nsMesg) { DatasetId datasetId = nsMesg.getDatasetId(); DistributedNamespace namespace = datasetRegistry.get(datasetId); if (namespace == null) { - throw new NoSuchElementException("Unable to find namespace with id %s for message: %s".formatted(datasetId, message) ); + throw new NoSuchElementException( + "Unable to find namespace with id %s for message: %s".formatted(datasetId, message)); } namespace.getJobManager().addSlowJob(job); - } - else if (toManagerNode instanceof SlowMessage slowMessage) { + } else if (toManagerNode instanceof SlowMessage slowMessage) { slowMessage.setProgressReporter(job.getProgressReporter()); jobManager.addSlowJob(job); - } - else { + } else { jobManager.addFastJob(job); } } @@ -107,8 +107,7 @@ public void start() throws IOException { public void stop() { try { acceptor.dispose(); - } - catch (RuntimeException e) { + } catch (RuntimeException e) { log.error("{} could not be closed", acceptor, e); } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java index 7bb3b557df1..725c13b7fab 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java @@ -61,23 +61,25 @@ public void sessionCreated(IoSession session) { @Override public void sessionOpened(IoSession session) { - NetworkSession networkSession = new NetworkSession(session, config.getCluster().getNetworkSessionMaxQueueLength()); + NetworkSession networkSession = new NetworkSession( + session, + config.getCluster().getNetworkSessionMaxQueueLength()); // Schedule ShardNode and Worker registration, so we don't block this thread which does the actual sending scheduler.schedule(() -> { - context = new NetworkMessageContext.ShardNodeNetworkContext(networkSession, workers, config, environment); - log.info("Connected to ManagerNode @ `{}`", session.getRemoteAddress()); - - // Authenticate with ManagerNode - context.send(new AddShardNode()); - - for (Worker w : workers.getWorkers().values()) { - w.setSession(networkSession); - WorkerInformation info = w.getInfo(); - log.info("Sending worker identity '{}'", info.getName()); - networkSession.send(new RegisterWorker(info)); - } - }, 0, TimeUnit.SECONDS + context = new NetworkMessageContext.ShardNodeNetworkContext(networkSession, workers, config, environment); + log.info("Connected to ManagerNode @ `{}`", session.getRemoteAddress()); + + // Authenticate with ManagerNode + context.send(new AddShardNode()); + + for (Worker w : workers.getWorkers().values()) { + w.setSession(networkSession); + WorkerInformation info = w.getInfo(); + log.info("Sending worker identity '{}'", info.getName()); + networkSession.send(new RegisterWorker(info)); + } + }, 0, TimeUnit.SECONDS ); @@ -86,13 +88,15 @@ public void sessionOpened(IoSession session) { private static void scheduleIdleLogger(ScheduledExecutorService scheduler, IoSession session, Duration timeout) { scheduler.scheduleAtFixedRate( - () -> { - final Duration elapsed = Duration.milliseconds(System.currentTimeMillis() - session.getLastIoTime()); - if (elapsed.compareTo(timeout) > 0) { - log.trace("No message sent or received since {}", elapsed); - } - }, - timeout.toSeconds(), timeout.toSeconds() / 2, TimeUnit.SECONDS + () -> { + final Duration elapsed = Duration.milliseconds(System.currentTimeMillis() - session.getLastIoTime()); + if (elapsed.compareTo(timeout) > 0) { + log.trace("No message sent or received since {}", elapsed); + } + }, + timeout.toSeconds(), + timeout.toSeconds() / 2, + TimeUnit.SECONDS ); } @@ -102,16 +106,15 @@ public void sessionClosed(IoSession session) { try { scheduler.schedule(this::connectToCluster, 2, TimeUnit.SECONDS); - } - catch (RejectedExecutionException e) { + } catch (RejectedExecutionException e) { log.trace("Scheduler rejected execution (probably in shutdown). Skipping reconnect attempt", e); } } private void connectToCluster() { final InetSocketAddress address = new InetSocketAddress( - config.getCluster().getManagerURL().getHostAddress(), - config.getCluster().getPort() + config.getCluster().getManagerURL().getHostAddress(), + config.getCluster().getPort() ); disconnectFromCluster(); @@ -135,11 +138,9 @@ private void connectToCluster() { // Sleep thirty seconds then retry. TimeUnit.SECONDS.sleep(config.getCluster().getConnectRetryTimeout().toSeconds()); - } - catch (RuntimeIoException e) { + } catch (RuntimeIoException e) { log.warn("Failed to connect to {}", address, e); - } - catch (InterruptedException e) { + } catch (InterruptedException e) { log.warn("Interrupted while trying to connector to cluster, giving up.", e); break; } @@ -188,14 +189,19 @@ public void messageReceived(IoSession session, Object message) { return; } - log.trace("{} received {} from {}", environment.getName(), message.getClass().getSimpleName(), session.getRemoteAddress()); - ReactingJob job = new ReactingJob<>((MessageToShardNode) message, context); + log.trace( + "{} received {} from {}", + environment.getName(), + message.getClass().getSimpleName(), + session.getRemoteAddress()); + ReactingJob job = new ReactingJob<>( + (MessageToShardNode) message, + context); if (message instanceof SlowMessage slowMessage) { slowMessage.setProgressReporter(job.getProgressReporter()); jobManager.addSlowJob(job); - } - else { + } else { jobManager.addFastJob(job); } } @@ -242,14 +248,14 @@ private void reportJobManagerStatus() { // Collect the ShardNode and all its workers jobs into a single queue for (Worker worker : workers.getWorkers().values()) { final JobManagerStatus jobManagerStatus = JobManagerStatus.builder() - .dataset(worker.getInfo().getDataset()) - .jobs(worker.getJobManager().getJobStatus()) - .build(); + .dataset( + worker.getInfo().getDataset()) + .jobs(worker.getJobManager().getJobStatus()) + .build(); try { context.trySend(new UpdateJobManagerStatus(jobManagerStatus)); - } - catch (Exception e) { + } catch (Exception e) { log.warn("Failed to report job manager status", e); if (config.isFailOnError()) { diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterEntityResolver.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterEntityResolver.java index 08fd8a60278..19853b9573a 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterEntityResolver.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterEntityResolver.java @@ -2,12 +2,12 @@ import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.*; +import jakarta.validation.constraints.NotEmpty; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Function; -import jakarta.validation.constraints.NotEmpty; import com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolver; import com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil; @@ -22,12 +22,12 @@ public class ClusterEntityResolver implements EntityResolver { @Override public ResolveStatistic resolveEntities( - @NotEmpty String[][] values, - List format, - EntityIdMap mapping, - IdColumnConfig idColumnConfig, - DateReader dateReader, - boolean onlySingles + @NotEmpty String[][] values, + List format, + EntityIdMap mapping, + IdColumnConfig idColumnConfig, + DateReader dateReader, + boolean onlySingles ) { final Map resolved = new HashMap<>(); final List unresolvedDate = new ArrayList<>(); @@ -40,7 +40,9 @@ public ResolveStatistic resolveEntities( // Row -> Column -> Value final Map[] extraDataByRow = EntityResolverUtil.readExtras(values, format); - final List> readers = IdColumnUtil.getIdReaders(format, idColumnConfig.getIdMappers()); + final List> readers = IdColumnUtil.getIdReaders( + format, + idColumnConfig.getIdMappers()); // We will not be able to resolve anything... if (readers.isEmpty()) { diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterImportHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterImportHandler.java index 3d697228be6..d1a56b86a7c 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterImportHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterImportHandler.java @@ -1,5 +1,9 @@ package com.bakdata.conquery.mode.cluster; +import jakarta.ws.rs.BadRequestException; +import jakarta.ws.rs.NotFoundException; +import jakarta.ws.rs.WebApplicationException; +import jakarta.ws.rs.core.Response; import java.io.IOException; import java.io.InputStream; import java.util.Collection; @@ -7,10 +11,6 @@ import java.util.List; import java.util.Map; import java.util.stream.Stream; -import jakarta.ws.rs.BadRequestException; -import jakarta.ws.rs.NotFoundException; -import jakarta.ws.rs.WebApplicationException; -import jakarta.ws.rs.core.Response; import com.bakdata.conquery.mode.ImportHandler; import com.bakdata.conquery.models.datasets.Import; @@ -64,7 +64,7 @@ private static void handleImport(Namespace namespace, InputStream inputStream, b readAndDistributeImport(((DistributedNamespace) namespace), table, header, parser); - try(Stream> allConcepts = namespace.getStorage().getAllConcepts()) { + try (Stream> allConcepts = namespace.getStorage().getAllConcepts()) { clearDependentConcepts(allConcepts, table.getId()); } } @@ -90,7 +90,8 @@ private static Table validateImportable(DistributedNamespace namespace, Preproce final String errorsMessage = String.join("\n - ", errors); log.error("Problems concerning Import `{}`:\n{}", importId, errorsMessage); - throw new BadRequestException("Headers[%s] do not match Table[%s]:\n%s".formatted(importId, tableId, errorsMessage)); + throw new BadRequestException( + "Headers[%s] do not match Table[%s]:\n%s".formatted(importId, tableId, errorsMessage)); } final Import processedImport = namespace.getStorage().getImport(importId); @@ -103,15 +104,20 @@ private static Table validateImportable(DistributedNamespace namespace, Preproce // before updating the import, make sure that all workers removed the prior import namespace.getWorkerHandler().sendToAll(new RemoveImportJob(importId)); namespace.getStorage().removeImport(importId); - } - else if (processedImport != null) { - throw new WebApplicationException("Import[%s] is already present.".formatted(importId), Response.Status.CONFLICT); + } else if (processedImport != null) { + throw new WebApplicationException( + "Import[%s] is already present.".formatted(importId), + Response.Status.CONFLICT); } return table; } - private static void readAndDistributeImport(DistributedNamespace namespace, Table table, PreprocessedHeader header, PreprocessedReader reader) { + private static void readAndDistributeImport( + DistributedNamespace namespace, + Table table, + PreprocessedHeader header, + PreprocessedReader reader) { final TableId tableId = table.getId(); log.info("BEGIN importing {} into {}", header.getName(), table); @@ -138,8 +144,8 @@ private static void readAndDistributeImport(DistributedNamespace namespace, Tabl final WorkerInformation responsibleWorker = namespace.getWorkerHandler().assignResponsibleWorker(bucketId); - sendBucket(bucket, responsibleWorker).addListener((f) -> { - if(((WriteFuture)f).isWritten()) { + sendBucket(bucket, responsibleWorker).addListener((f) -> { + if (((WriteFuture) f).isWritten()) { log.trace("Sent Bucket {}", bucketId); return; } @@ -157,7 +163,11 @@ private static void readAndDistributeImport(DistributedNamespace namespace, Tabl namespace.getJobManager().addSlowJob(new RegisterImportEntities(collectedEntities, namespace, imp.getId())); - log.debug("Successfully read {} Buckets, containing {} entities for `{}`", header.getNumberOfBuckets(), header.getNumberOfEntities(), imp.getId()); + log.debug( + "Successfully read {} Buckets, containing {} entities for `{}`", + header.getNumberOfBuckets(), + header.getNumberOfEntities(), + imp.getId()); namespace.getWorkerHandler().sendUpdatedWorkerInformation(); @@ -165,10 +175,11 @@ private static void readAndDistributeImport(DistributedNamespace namespace, Tabl private static void clearDependentConcepts(Stream> allConcepts, TableId table) { allConcepts.map(Concept::getConnectors) - .flatMap(List::stream) - .filter(con -> con.resolveTableId().equals(table)) - .map(Connector::getConcept) - .forEach(Concept::clearMatchingStats); + .flatMap(List::stream) + .filter( + con -> con.resolveTableId().equals(table)) + .map(Connector::getConcept) + .forEach(Concept::clearMatchingStats); } /** @@ -192,7 +203,7 @@ public void deleteImport(ImportId imp) { final DatasetId id = imp.getTable().getDataset(); final DistributedNamespace namespace = datasetRegistry.get(id); - try(Stream> allConcepts = namespace.getStorage().getAllConcepts()) { + try (Stream> allConcepts = namespace.getStorage().getAllConcepts()) { clearDependentConcepts(allConcepts, imp.getTable()); } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java index 1403bc8f5e8..2361ebc2c75 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java @@ -25,25 +25,47 @@ public class ClusterManagerProvider implements ManagerProvider { public ClusterManager provideManager(ConqueryConfig config, Environment environment) { final JobManager jobManager = ManagerProvider.newJobManager(config); final MetaStorage storage = new MetaStorage(config.getStorage()); - final InternalMapperFactory internalMapperFactory = new InternalMapperFactory(config, environment.getValidator()); + final InternalMapperFactory internalMapperFactory = new InternalMapperFactory( + config, + environment.getValidator()); final ClusterState clusterState = new ClusterState(); - final NamespaceHandler namespaceHandler = new ClusterNamespaceHandler(clusterState, config, internalMapperFactory); - final DatasetRegistry datasetRegistry = ManagerProvider.createDatasetRegistry(namespaceHandler, config, internalMapperFactory); + final NamespaceHandler namespaceHandler = new ClusterNamespaceHandler( + clusterState, + config, + internalMapperFactory); + final DatasetRegistry datasetRegistry = ManagerProvider.createDatasetRegistry( + namespaceHandler, + config, + internalMapperFactory); - final ClusterConnectionManager connectionManager = - new ClusterConnectionManager(datasetRegistry, jobManager, config, internalMapperFactory, clusterState); + final ClusterConnectionManager connectionManager = new ClusterConnectionManager( + datasetRegistry, + jobManager, + config, + internalMapperFactory, + clusterState); final ImportHandler importHandler = new ClusterImportHandler(datasetRegistry); final StorageListener extension = new ClusterStorageListener(jobManager, datasetRegistry); final Supplier> nodeProvider = () -> clusterState.getShardNodes().values(); final List adminTasks = List.of(new ReportConsistencyTask(clusterState)); - final DelegateManager - delegate = - new DelegateManager<>(config, environment, datasetRegistry, storage, importHandler, extension, nodeProvider, adminTasks, internalMapperFactory, jobManager); + final DelegateManager delegate = new DelegateManager<>( + config, + environment, + datasetRegistry, + storage, + importHandler, + extension, + nodeProvider, + adminTasks, + internalMapperFactory, + jobManager); environment.healthChecks() - .register("cluster", new ClusterHealthCheck(clusterState, config.getCluster().getHeartbeatTimeout().toJavaDuration())); + .register( + "cluster", + new ClusterHealthCheck(clusterState, config.getCluster().getHeartbeatTimeout().toJavaDuration())); return new ClusterManager(delegate, connectionManager); } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java index 7d7f2244fb6..1c537ba4602 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java @@ -24,25 +24,34 @@ public class ClusterNamespaceHandler implements NamespaceHandler datasetRegistry, - Environment environment) { + NamespaceStorage namespaceStorage, + MetaStorage metaStorage, + DatasetRegistry datasetRegistry, + Environment environment) { - NamespaceSetupData namespaceData = NamespaceHandler.createNamespaceSetup(namespaceStorage, config, internalMapperFactory, datasetRegistry, environment); - DistributedExecutionManager executionManager = new DistributedExecutionManager(metaStorage, datasetRegistry, clusterState, config); + NamespaceSetupData namespaceData = NamespaceHandler.createNamespaceSetup( + namespaceStorage, + config, + internalMapperFactory, + datasetRegistry, + environment); + DistributedExecutionManager executionManager = new DistributedExecutionManager( + metaStorage, + datasetRegistry, + clusterState, + config); WorkerHandler workerHandler = new WorkerHandler(namespaceStorage); clusterState.getWorkerHandlers().put(namespaceStorage.getDataset().getId(), workerHandler); DistributedNamespace distributedNamespace = new DistributedNamespace( - namespaceData.preprocessMapper(), - namespaceStorage, - executionManager, - namespaceData.jobManager(), - namespaceData.filterSearch(), - new ClusterEntityResolver(), - workerHandler, - config.getCluster() + namespaceData.preprocessMapper(), + namespaceStorage, + executionManager, + namespaceData.jobManager(), + namespaceData.filterSearch(), + new ClusterEntityResolver(), + workerHandler, + config.getCluster() ); for (ShardNodeInformation node : clusterState.getShardNodes().values()) { @@ -54,7 +63,10 @@ public DistributedNamespace createNamespace( @Override public void removeNamespace(DatasetId id, DistributedNamespace namespace) { - clusterState.getShardNodes().values().forEach(shardNode -> shardNode.send(new RemoveWorker(namespace.getDataset().getId()))); + clusterState.getShardNodes() + .values() + .forEach( + shardNode -> shardNode.send(new RemoveWorker(namespace.getDataset().getId()))); clusterState.getWorkerHandlers().keySet().removeIf(worker -> worker.getDataset().getDataset().equals(id)); } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterState.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterState.java index 18e725f1d05..c5541f6c989 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterState.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterState.java @@ -21,14 +21,20 @@ public class ClusterState { public synchronized void register(ShardNodeInformation node, WorkerInformation info) { WorkerHandler workerHandler = workerHandlers.get(info.getDataset()); if (workerHandler == null) { - throw new NoSuchElementException("Trying to register a worker for unknown dataset '%s'. I only know %s".formatted(info.getDataset(), workerHandlers.keySet())); + throw new NoSuchElementException( + "Trying to register a worker for unknown dataset '%s'. I only know %s".formatted( + info.getDataset(), + workerHandlers.keySet())); } workerHandler.register(node, info); } public WorkerInformation getWorker(final WorkerId workerId, final DatasetId id) { return Optional.ofNullable(workerHandlers.get(id)) - .flatMap(ns -> ns.getWorkers().getOptional(workerId)) - .orElseThrow(() -> new NoSuchElementException("Unknown worker worker '%s' for dataset '%s'".formatted(workerId, id))); + .flatMap( + ns -> ns.getWorkers().getOptional(workerId)) + .orElseThrow( + () -> new NoSuchElementException( + "Unknown worker worker '%s' for dataset '%s'".formatted(workerId, id))); } } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterStorageListener.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterStorageListener.java index 46d3f361944..1ffbfc08195 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterStorageListener.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterStorageListener.java @@ -22,16 +22,24 @@ /** * Propagates changes of stored entities to relevant ConQuery shards in the cluster. */ -public record ClusterStorageListener(JobManager jobManager, DatasetRegistry datasetRegistry) implements StorageListener { +public record ClusterStorageListener( + JobManager jobManager, + DatasetRegistry datasetRegistry) implements StorageListener { @Override public void onAddSecondaryId(SecondaryIdDescription secondaryId) { - datasetRegistry().get(secondaryId.getDataset()).getWorkerHandler().sendToAll(new UpdateSecondaryId(secondaryId)); + datasetRegistry().get(secondaryId.getDataset()) + .getWorkerHandler() + .sendToAll( + new UpdateSecondaryId(secondaryId)); } @Override public void onDeleteSecondaryId(SecondaryIdDescriptionId secondaryId) { - datasetRegistry().get(secondaryId.getDataset()).getWorkerHandler().sendToAll(new RemoveSecondaryId(secondaryId)); + datasetRegistry().get(secondaryId.getDataset()) + .getWorkerHandler() + .sendToAll( + new RemoveSecondaryId(secondaryId)); } @Override @@ -47,14 +55,18 @@ public void onRemoveTable(TableId table) { @Override public void onAddConcept(Concept concept) { WorkerHandler handler = datasetRegistry().get(concept.getDataset()).getWorkerHandler(); - SimpleJob simpleJob = new SimpleJob(String.format("sendToAll : Add %s ", concept.getId()), () -> handler.sendToAll(new UpdateConcept(concept))); + SimpleJob simpleJob = new SimpleJob( + String.format("sendToAll : Add %s ", concept.getId()), + () -> handler.sendToAll(new UpdateConcept(concept))); jobManager().addSlowJob(simpleJob); } @Override public void onDeleteConcept(ConceptId concept) { WorkerHandler handler = datasetRegistry().get(concept.getDataset()).getWorkerHandler(); - SimpleJob simpleJob = new SimpleJob("sendToAll: remove " + concept, () -> handler.sendToAll(new RemoveConcept(concept))); + SimpleJob simpleJob = new SimpleJob( + "sendToAll: remove " + concept, + () -> handler.sendToAll(new RemoveConcept(concept))); jobManager().addSlowJob(simpleJob); } } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/InternalMapperFactory.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/InternalMapperFactory.java index ea396d87938..93ceedeaa92 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/InternalMapperFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/InternalMapperFactory.java @@ -21,7 +21,8 @@ public record InternalMapperFactory(ConqueryConfig config, Validator validator) * @return a preconfigured binary object mapper */ private ObjectMapper createInternalObjectMapper(Class viewClass) { - final ObjectMapper objectMapper = config.configureObjectMapper(Jackson.copyMapperAndInjectables(Jackson.BINARY_MAPPER)); + final ObjectMapper objectMapper = config.configureObjectMapper( + Jackson.copyMapperAndInjectables(Jackson.BINARY_MAPPER)); final MutableInjectableValues injectableValues = new MutableInjectableValues(); objectMapper.setInjectableValues(injectableValues); @@ -61,7 +62,9 @@ public ObjectMapper createWorkerPersistenceMapper(NamespacedStorageProvider work return objectMapper; } - public ObjectMapper createNamespacePersistenceMapper(NamespaceStorage namespaceStorage, DatasetRegistry datasetRegistry) { + public ObjectMapper createNamespacePersistenceMapper( + NamespaceStorage namespaceStorage, + DatasetRegistry datasetRegistry) { final ObjectMapper objectMapper = createInternalObjectMapper(View.Persistence.Manager.class); namespaceStorage.injectInto(objectMapper); @@ -108,7 +111,10 @@ public ObjectMapper createPreprocessMapper(NamespaceStorage namespaceStorage, Da * * @param objectMapper to be configured (should be a JSON mapper) */ - public void customizeApiObjectMapper(ObjectMapper objectMapper, DatasetRegistry datasetRegistry, MetaStorage metaStorage) { + public void customizeApiObjectMapper( + ObjectMapper objectMapper, + DatasetRegistry datasetRegistry, + MetaStorage metaStorage) { InternalMapperFactory.setViewClass(objectMapper, View.Api.class); diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/ConnectionManager.java b/backend/src/main/java/com/bakdata/conquery/mode/local/ConnectionManager.java index 45c02eec1f1..51f6050e70a 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/ConnectionManager.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/ConnectionManager.java @@ -24,7 +24,11 @@ public DSLContext connect(Dataset dataset) { } public ManagedConnection getConnection(Dataset dataset) { - return Objects.requireNonNull(connections.get(dataset.getDataSource()), () -> "No connection available for datset=%s, dataSource=%s".formatted(dataset.getName(), dataset.getDataSource())); + return Objects.requireNonNull( + connections.get(dataset.getDataSource()), + () -> "No connection available for datset=%s, dataSource=%s".formatted( + dataset.getName(), + dataset.getDataSource())); } } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java index fe9d62d8ce3..ec0d7460d9a 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java @@ -33,21 +33,30 @@ public DelegateManager provideManager(ConqueryConfig config, Env final JobManager jobManager = ManagerProvider.newJobManager(config); final MetaStorage storage = new MetaStorage(config.getStorage()); - final InternalMapperFactory internalMapperFactory = new InternalMapperFactory(config, environment.getValidator()); - final NamespaceHandler namespaceHandler = new LocalNamespaceHandler(config, internalMapperFactory, connectionManager, clock); - final DatasetRegistry datasetRegistry = ManagerProvider.createDatasetRegistry(namespaceHandler, config, internalMapperFactory); + final InternalMapperFactory internalMapperFactory = new InternalMapperFactory( + config, + environment.getValidator()); + final NamespaceHandler namespaceHandler = new LocalNamespaceHandler( + config, + internalMapperFactory, + connectionManager, + clock); + final DatasetRegistry datasetRegistry = ManagerProvider.createDatasetRegistry( + namespaceHandler, + config, + internalMapperFactory); return new DelegateManager<>( - config, - environment, - datasetRegistry, - storage, - new FailingImportHandler(), - new LocalStorageListener(jobManager, datasetRegistry), - EMPTY_NODE_PROVIDER, - List.of(), - internalMapperFactory, - ManagerProvider.newJobManager(config) + config, + environment, + datasetRegistry, + storage, + new FailingImportHandler(), + new LocalStorageListener(jobManager, datasetRegistry), + EMPTY_NODE_PROVIDER, + List.of(), + internalMapperFactory, + ManagerProvider.newJobManager(config) ); } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java index 8a4a292b617..baddcf8c4e0 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.mode.local; +import java.time.Clock; + import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.mode.NamespaceHandler; @@ -21,8 +23,6 @@ import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; -import java.time.Clock; - @RequiredArgsConstructor @Slf4j public class LocalNamespaceHandler implements NamespaceHandler { @@ -34,12 +34,17 @@ public class LocalNamespaceHandler implements NamespaceHandler { @Override public LocalNamespace createNamespace( - NamespaceStorage namespaceStorage, - MetaStorage metaStorage, - DatasetRegistry datasetRegistry, - Environment environment) { + NamespaceStorage namespaceStorage, + MetaStorage metaStorage, + DatasetRegistry datasetRegistry, + Environment environment) { - NamespaceSetupData namespaceData = NamespaceHandler.createNamespaceSetup(namespaceStorage, config, internalMapperFactory, datasetRegistry, environment); + NamespaceSetupData namespaceData = NamespaceHandler.createNamespaceSetup( + namespaceStorage, + config, + internalMapperFactory, + datasetRegistry, + environment); ManagedConnection connection = connectionManager.getConnection(namespaceStorage.getDataset()); try { @@ -49,22 +54,38 @@ public LocalNamespace createNamespace( ResultSetProcessor resultSetProcessor = dialectBundle.getResultSetProcessor(config); SqlExecutionService sqlExecutionService = new SqlExecutionService(dslContext, resultSetProcessor); - NodeConversions nodeConversions = new NodeConversions(config.getIdColumns(), dialectBundle, dslContext, sqlExecutionService, clock, connection.getConnection().getPrimaryColumn()); + NodeConversions nodeConversions = new NodeConversions( + config.getIdColumns(), + dialectBundle, + dslContext, + sqlExecutionService, + clock, + connection.getConnection().getPrimaryColumn()); SqlConverter sqlConverter = new SqlConverter(nodeConversions, config); - ExecutionManager executionManager = new SqlExecutionManager(sqlConverter, sqlExecutionService, metaStorage, datasetRegistry, config); + ExecutionManager executionManager = new SqlExecutionManager( + sqlConverter, + sqlExecutionService, + metaStorage, + datasetRegistry, + config); SqlStorageHandler sqlStorageHandler = new SqlStorageHandler(sqlExecutionService); - SqlEntityResolver sqlEntityResolver = new SqlEntityResolver(config.getIdColumns(), dslContext, dialectBundle, sqlExecutionService); + SqlEntityResolver sqlEntityResolver = new SqlEntityResolver( + config.getIdColumns(), + dslContext, + dialectBundle, + sqlExecutionService); return new LocalNamespace( - dialectBundle, - namespaceData.preprocessMapper(), - namespaceStorage, - executionManager, - dslContext, sqlStorageHandler, - namespaceData.jobManager(), - namespaceData.filterSearch(), - sqlEntityResolver, - connection.getConnection() + dialectBundle, + namespaceData.preprocessMapper(), + namespaceStorage, + executionManager, + dslContext, + sqlStorageHandler, + namespaceData.jobManager(), + namespaceData.filterSearch(), + sqlEntityResolver, + connection.getConnection() ); } catch (Exception e) { log.error("Failed to load namespaceStorage for {}", namespaceStorage.getPathName(), e); diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalStorageListener.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalStorageListener.java index 0d8091d4251..fdf8b6fff91 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalStorageListener.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalStorageListener.java @@ -12,7 +12,9 @@ import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.LocalNamespace; -public record LocalStorageListener(JobManager jobManager, DatasetRegistry datasetRegistry) implements StorageListener { +public record LocalStorageListener( + JobManager jobManager, + DatasetRegistry datasetRegistry) implements StorageListener { @Override public void onAddSecondaryId(SecondaryIdDescription secondaryId) { diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/ManagedConnection.java b/backend/src/main/java/com/bakdata/conquery/mode/local/ManagedConnection.java index 0e1bf735f77..d42c5152cef 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/ManagedConnection.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/ManagedConnection.java @@ -36,28 +36,31 @@ public void start() throws Exception { if (dataSource.getConnection().isValid(1000)) { log.info("SUCCESS connecting to {}", connection.getJdbcConnectionUrl()); } else { - log.error("FAILED connecting to {}. Connection did not become valid.", connection.getJdbcConnectionUrl()); + log.error( + "FAILED connecting to {}. Connection did not become valid.", + connection.getJdbcConnectionUrl()); } } catch (SQLException exception) { - throw new RuntimeException("FAILED connecting to %s".formatted(connection.getJdbcConnectionUrl()), exception); + throw new RuntimeException( + "FAILED connecting to %s".formatted(connection.getJdbcConnectionUrl()), + exception); } } public DSLContext connect() { Preconditions.checkNotNull(this.dataSource, "dataSource has not been initialized yet."); - Settings settings = new Settings() - .withRenderFormatted(config.isWithPrettyPrinting()) - // enforces all identifiers to be quoted if not explicitly unquoted via DSL.unquotedName() - // to prevent any lowercase/uppercase SQL dialect specific identifier naming issues - .withRenderQuotedNames(RenderQuotedNames.EXPLICIT_DEFAULT_QUOTED) - // always render "as" keyword for field aliases - .withRenderOptionalAsKeywordForFieldAliases(RenderOptionalKeyword.ON); + Settings settings = new Settings().withRenderFormatted(config.isWithPrettyPrinting()) + // enforces all identifiers to be quoted if not explicitly unquoted via DSL.unquotedName() + // to prevent any lowercase/uppercase SQL dialect specific identifier naming issues + .withRenderQuotedNames(RenderQuotedNames.EXPLICIT_DEFAULT_QUOTED) + // always render "as" keyword for field aliases + .withRenderOptionalAsKeywordForFieldAliases(RenderOptionalKeyword.ON); return DSL.using( - this.dataSource, - connection.getDialect().getDialectBundle().getJooqDialect(), - settings + this.dataSource, + connection.getDialect().getDialectBundle().getJooqDialect(), + settings ); } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java b/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java index 8768f24c983..290e8c174fc 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java @@ -3,13 +3,13 @@ import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.*; import static org.jooq.impl.DSL.*; +import jakarta.validation.constraints.NotEmpty; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; -import jakarta.validation.constraints.NotEmpty; import com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolver; import com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil; @@ -33,7 +33,6 @@ import org.jooq.Select; import org.jooq.SelectConditionStep; import org.jooq.Table; -import org.jooq.impl.DSL; @RequiredArgsConstructor public class SqlEntityResolver implements EntityResolver { @@ -48,12 +47,12 @@ public class SqlEntityResolver implements EntityResolver { @Override public ResolveStatistic resolveEntities( - @NotEmpty String[][] values, - List format, - EntityIdMap mapping, - IdColumnConfig idColumnConfig, - DateReader dateReader, - boolean onlySingles + @NotEmpty String[][] values, + List format, + EntityIdMap mapping, + IdColumnConfig idColumnConfig, + DateReader dateReader, + boolean onlySingles ) { final Map resolved = new HashMap<>(); final List unresolvedDate = new ArrayList<>(); @@ -66,7 +65,9 @@ public ResolveStatistic resolveEntities( // Row -> Column -> Value final Map[] extraDataByRow = EntityResolverUtil.readExtras(values, format); - final List> readers = IdColumnUtil.getIdReaders(format, idColumnConfig.getIdMappers()); + final List> readers = IdColumnUtil.getIdReaders( + format, + idColumnConfig.getIdMappers()); // We will not be able to resolve anything... if (readers.isEmpty()) { @@ -146,24 +147,38 @@ private Map resolveIds(String[][] values, List externalPrimaryColumn = field(name(SharedAliases.PRIMARY_COLUMN.getAlias()), String.class); Field innerPrimaryColumn = field(name(idColumns.findPrimaryIdColumn().getField()), String.class); // Would prefer this to be `is not null`, but hana does not support that for fields - Field isResolved = case_().when(innerPrimaryColumn.isNull(), inline(false)).otherwise(inline(true)).as(IS_RESOLVED_ALIAS); + Field isResolved = case_().when(innerPrimaryColumn.isNull(), inline(false)) + .otherwise(inline(true)) + .as( + IS_RESOLVED_ALIAS); Table allIdsTable = table(name(idColumns.getTable())); - SelectConditionStep> resolveIdsQuery = - context.with(unresolvedCte) - .select(rowIndex, externalPrimaryColumn, isResolved) - .from(dialect.getFunctionProvider().innerJoin(allIdsTable, unresolvedCte, List.of(externalPrimaryColumn.eq(innerPrimaryColumn)))) - .where(externalPrimaryColumn.eq(innerPrimaryColumn)); + SelectConditionStep> resolveIdsQuery = context.with(unresolvedCte) + .select( + rowIndex, + externalPrimaryColumn, + isResolved) + .from( + dialect.getFunctionProvider() + .innerJoin( + allIdsTable, + unresolvedCte, + List.of(externalPrimaryColumn.eq(innerPrimaryColumn)))) + .where( + externalPrimaryColumn.eq(innerPrimaryColumn)); return executionService.fetchStream(resolveIdsQuery) - .collect(Collectors.toMap( - record -> record.get(rowIndex), - record -> new IdResolveInfo(record.get(externalPrimaryColumn), record.get(isResolved)) - )); + .collect( + Collectors.toMap( + record -> record.get(rowIndex), + record -> new IdResolveInfo(record.get(externalPrimaryColumn), record.get(isResolved)) + )); } - private CommonTableExpression createUnresolvedCte(String[][] values, List> readers) { + private CommonTableExpression createUnresolvedCte( + String[][] values, + List> readers) { List>> selects = new ArrayList<>(values.length); for (int i = 1; i < values.length; i++) { @@ -184,8 +199,8 @@ private CommonTableExpression createUnresolvedCte(String[][] values, List rowIndex = inline(i).as(ROW_INDEX); Field externalPrimaryColumn = inline(resolvedId).as(SharedAliases.PRIMARY_COLUMN.getAlias()); Select> externalIdSelect = context.select(rowIndex, externalPrimaryColumn) - // some dialects can't just select static values without FROM clause - .from(dialect.getFunctionProvider().getNoOpTable()); + // some dialects can't just select static values without FROM clause + .from(dialect.getFunctionProvider().getNoOpTable()); selects.add(externalIdSelect); } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/SqlStorageHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/local/SqlStorageHandler.java index 5e3141f1b0f..a186bf58e17 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/SqlStorageHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/SqlStorageHandler.java @@ -25,20 +25,22 @@ public SqlStorageHandler(SqlExecutionService sqlExecutionService) { @Override public Stream lookupColumnValues(NamespaceStorage namespaceStorage, Column column) { - Select> columValuesQuery = dslContext.selectDistinct(DSL.field(DSL.name(column.getName()))) - .from(DSL.table(DSL.name(column.getTable().getName()))); + Select> columValuesQuery = dslContext.selectDistinct( + DSL.field(DSL.name(column.getName()))).from(DSL.table(DSL.name(column.getTable().getName()))); return queryForDistinctValues(columValuesQuery); } private Stream queryForDistinctValues(Select> columValuesQuery) { try { return sqlExecutionService.fetchStream(columValuesQuery) - .map(record -> record.get(0, String.class)) - // the database might return null or a blank string as a distinct value - .filter(value -> value != null && !value.isBlank()); - } - catch (Exception e) { - log.error("Expecting exactly 1 column in Result when querying for distinct values of a column. Query: {}.", columValuesQuery, e); + .map(record -> record.get(0, String.class)) + // the database might return null or a blank string as a distinct value + .filter(value -> value != null && !value.isBlank()); + } catch (Exception e) { + log.error( + "Expecting exactly 1 column in Result when querying for distinct values of a column. Query: {}.", + columValuesQuery, + e); } return Stream.empty(); } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/UpdateMatchingStatsSqlJob.java b/backend/src/main/java/com/bakdata/conquery/mode/local/UpdateMatchingStatsSqlJob.java index 945e1171ac4..60556265026 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/UpdateMatchingStatsSqlJob.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/UpdateMatchingStatsSqlJob.java @@ -43,21 +43,26 @@ public void execute() throws Exception { Stopwatch stopwatch = Stopwatch.createStarted(); - ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(getMatchingStats().getMatchingStatsWorkers())); + ListeningExecutorService executorService = MoreExecutors.listeningDecorator( + Executors.newFixedThreadPool(getMatchingStats().getMatchingStatsWorkers())); Map> jobsByConcept = new HashedMap<>(); Collection> jobs = jobsByConcept.values(); for (Concept concept : concepts) { if (concept instanceof TreeConcept) { - ListenableFuture job = matchingStats.collectMatchingStatsForConcept((TreeConcept) concept, executorService, getMatchingStats().getMatchingStatsRetries()); + ListenableFuture job = matchingStats.collectMatchingStatsForConcept( + (TreeConcept) concept, + executorService, + getMatchingStats().getMatchingStatsRetries()); job.addListener( - () -> { - if (job.state().equals(Future.State.FAILED)) { - log.warn("FAILED to collect SQL matching stats for {}", concept, job.exceptionNow()); - } - }, MoreExecutors.directExecutor()); + () -> { + if (job.state().equals(Future.State.FAILED)) { + log.warn("FAILED to collect SQL matching stats for {}", concept, job.exceptionNow()); + } + }, + MoreExecutors.directExecutor()); jobsByConcept.put(concept.getId(), job); } @@ -81,7 +86,9 @@ public void execute() throws Exception { // intentionally left blank } - log.debug("WAITING for {} matching stats to finish.", jobs.stream().filter(job -> job.state().equals(Future.State.RUNNING)).count()); + log.debug( + "WAITING for {} matching stats to finish.", + jobs.stream().filter(job -> job.state().equals(Future.State.RUNNING)).count()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java b/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java index 72b3f186b2a..414c55d7dd5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java @@ -79,7 +79,11 @@ public final class AuthorizationController implements Managed { @Getter private DropwizardResourceConfig unprotectedAuthAdmin; - public AuthorizationController(@NotNull MetaStorage storage, @NotNull ConqueryConfig config, @NotNull Environment environment, AdminServlet adminServlet) { + public AuthorizationController( + @NotNull MetaStorage storage, + @NotNull ConqueryConfig config, + @NotNull Environment environment, + AdminServlet adminServlet) { this.storage = storage; this.config = config; this.environment = environment; @@ -100,8 +104,16 @@ protected void configure() { AuthFilter.registerTokenExtractor(JWTokenHandler.JWTokenExtractor.class, adminServlet.getJerseyConfigUI()); } - unprotectedAuthAdmin = AuthServlet.generalSetup(environment.metrics(), config, environment.admin(), environment.getObjectMapper()); - unprotectedAuthApi = AuthServlet.generalSetup(environment.metrics(), config, environment.servlets(), environment.getObjectMapper()); + unprotectedAuthAdmin = AuthServlet.generalSetup( + environment.metrics(), + config, + environment.admin(), + environment.getObjectMapper()); + unprotectedAuthApi = AuthServlet.generalSetup( + environment.metrics(), + config, + environment.servlets(), + environment.getObjectMapper()); // Add the user token realm @@ -155,7 +167,9 @@ private void externalInit() { } // Register all realms in Shiro - log.info("Registering the following realms to Shiro:\n\t{}", realms.stream().map(Realm::getName).collect(Collectors.joining("\n\t"))); + log.info( + "Registering the following realms to Shiro:\n\t{}", + realms.stream().map(Realm::getName).collect(Collectors.joining("\n\t"))); securityManager.setRealms(realms); } @@ -165,7 +179,10 @@ private void externalInit() { * * @param storage A storage, where the handler might add a new users. */ - private static void initializeAuthConstellation(@NonNull AuthorizationConfig config, @NonNull List realms, @NonNull MetaStorage storage) { + private static void initializeAuthConstellation( + @NonNull AuthorizationConfig config, + @NonNull List realms, + @NonNull MetaStorage storage) { for (ProtoRole pRole : config.getInitialRoles()) { pRole.createOrOverwriteRole(storage); } @@ -176,7 +193,11 @@ private static void initializeAuthConstellation(@NonNull AuthorizationConfig con for (Realm realm : realms) { if (realm instanceof UserManageable) { - AuthorizationHelper.registerForAuthentication((UserManageable) realm, user, pUser.getCredential(), true); + AuthorizationHelper.registerForAuthentication( + (UserManageable) realm, + user, + pUser.getCredential(), + true); } } } @@ -226,18 +247,20 @@ public static User flatCopyUser(@NonNull User originUser, String namePrefix, @No // Give read permission to all executions the original user owned copiedPermission.addAll( - storage.getAllExecutions() - .filter(originUser::isOwner) - .map(exc -> exc.createPermission(Ability.READ.asSet())) - .collect(Collectors.toSet()) + storage.getAllExecutions() + .filter(originUser::isOwner) + .map( + exc -> exc.createPermission(Ability.READ.asSet())) + .collect(Collectors.toSet()) ); // Give read permission to all form configs the original user owned copiedPermission.addAll( - storage.getAllFormConfigs() - .filter(originUser::isOwner) - .map(conf -> conf.createPermission(Ability.READ.asSet())) - .collect(Collectors.toSet()) + storage.getAllFormConfigs() + .filter(originUser::isOwner) + .map( + conf -> conf.createPermission(Ability.READ.asSet())) + .collect(Collectors.toSet()) ); // Create copied user diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationHelper.java b/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationHelper.java index a6ec61e706a..4c3a544c1fa 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationHelper.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationHelper.java @@ -33,12 +33,12 @@ @UtilityClass public class AuthorizationHelper { - public static List getGroupsOf(@NonNull Subject subject, @NonNull MetaStorage storage){ - try(Stream allGroups = storage.getAllGroups()) { - return allGroups - .filter(g -> g.getMembers().contains(subject.getId())) - .sorted() - .collect(Collectors.toList()); + public static List getGroupsOf(@NonNull Subject subject, @NonNull MetaStorage storage) { + try (Stream allGroups = storage.getAllGroups()) { + return allGroups.filter(g -> g.getMembers().contains(subject.getId())) + .sorted() + .collect( + Collectors.toList()); } } @@ -48,7 +48,7 @@ public static List getGroupsOf(@NonNull Subject subject, @NonNull MetaSto */ public static Optional getPrimaryGroup(@NonNull Subject subject, @NonNull MetaStorage storage) { List groups = getGroupsOf(subject, storage); - if(groups.isEmpty()) { + if (groups.isEmpty()) { return Optional.empty(); } // TODO: 17.02.2020 implement primary flag for group @@ -60,13 +60,16 @@ public static Optional getPrimaryGroup(@NonNull Subject subject, @NonNull * the permission of the roles it inherits. The query can be filtered by the Permission domain. * @return Owned and inherited permissions. */ - public static Multimap getEffectiveUserPermissions(User user, List domainSpecifier, MetaStorage storage) { + public static Multimap getEffectiveUserPermissions( + User user, + List domainSpecifier, + MetaStorage storage) { Set permissions = user.getEffectivePermissions(); Multimap mappedPerms = ArrayListMultimap.create(); - for(ConqueryPermission perm : permissions) { + for (ConqueryPermission perm : permissions) { Set domains = perm.getDomains(); - if(!Collections.disjoint(domainSpecifier, perm.getDomains())) { - for(String domain : domains) { + if (!Collections.disjoint(domainSpecifier, perm.getDomains())) { + for (String domain : domains) { mappedPerms.put(domain, perm); } } @@ -90,18 +93,22 @@ public static void authorizeDownloadDatasets(@NonNull Subject subject, @NonNull NamespacedIdentifiableCollector collector = new NamespacedIdentifiableCollector(); visitable.visit(collector); - Set datasets = - collector.getIdentifiables() - .stream() - .map(NamespacedIdentifiable::getDataset) - .collect(Collectors.toSet()); + Set datasets = collector.getIdentifiables() + .stream() + .map(NamespacedIdentifiable::getDataset) + .collect( + Collectors.toSet()); subject.authorize(datasets, Ability.DOWNLOAD); } - public static boolean registerForAuthentication(UserManageable userManager, User user, CredentialType credentials, boolean override) { - if(override) { + public static boolean registerForAuthentication( + UserManageable userManager, + User user, + CredentialType credentials, + boolean override) { + if (override) { return userManager.updateUser(user.getId(), credentials); } return userManager.addUser(user.getId(), credentials); diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/ConqueryAuthenticationInfo.java b/backend/src/main/java/com/bakdata/conquery/models/auth/ConqueryAuthenticationInfo.java index bb97cec09c7..d05248768fd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/ConqueryAuthenticationInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/ConqueryAuthenticationInfo.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.models.auth; import java.net.URI; - import javax.annotation.Nullable; import com.bakdata.conquery.models.auth.entities.Subject; @@ -23,13 +22,13 @@ public class ConqueryAuthenticationInfo implements AuthenticationInfo { private final SubjectPrincipalCollection principals; - + /** * The credential a realm used for authentication. */ private final Object credentials; - - + + /** * A realm can indicate whether a logout button is shown for the user or not */ @@ -42,11 +41,11 @@ public class ConqueryAuthenticationInfo implements AuthenticationInfo { private final URI frontChannelLogout; public ConqueryAuthenticationInfo( - Subject subject, - Object credentials, - ConqueryAuthenticationRealm realm, - boolean displayLogout, - @Nullable URI frontChannelLogout + Subject subject, + Object credentials, + ConqueryAuthenticationRealm realm, + boolean displayLogout, + @Nullable URI frontChannelLogout ) { this.credentials = credentials; this.displayLogout = displayLogout; diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/ConqueryAuthenticator.java b/backend/src/main/java/com/bakdata/conquery/models/auth/ConqueryAuthenticator.java index 43a8f43ea42..28a5a654da0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/ConqueryAuthenticator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/ConqueryAuthenticator.java @@ -1,42 +1,44 @@ -package com.bakdata.conquery.models.auth; - -import java.util.Optional; - -import com.bakdata.conquery.models.auth.entities.Subject; -import com.bakdata.conquery.models.auth.web.AuthenticationExceptionMapper; -import com.bakdata.conquery.util.io.ConqueryMDC; -import io.dropwizard.auth.Authenticator; -import lombok.extern.slf4j.Slf4j; -import org.apache.shiro.SecurityUtils; -import org.apache.shiro.authc.AuthenticationToken; - -/** - * This dropwizard authenticator and the shiro realm are conceptually the same. - * They authenticate -- but shiro realms can also be used for authorization. - * We use this single authenticator to set up shiro and forward all requests to - * shiro, where multiple realms might be configured. - * We need this authenticator to plug in the security, and hereby shiro, into the AuthFilter. - */ -@Slf4j -public class ConqueryAuthenticator implements Authenticator{ - - /** - * The execeptions thrown by Shiro will be catched by {@link AuthenticationExceptionMapper}. - */ - @Override - public Optional authenticate(AuthenticationToken token) { - // Submit the token to Shiro (to all realms that were registered) - ConqueryAuthenticationInfo info = (ConqueryAuthenticationInfo) SecurityUtils.getSecurityManager().authenticate(token); - - // Extract - Subject subject = info.getPrincipals().oneByType(Subject.class); - - - // If the subject was present, all further authorization can now be performed on the subject object - log.trace("Using subject {} for further authorization", subject); - ConqueryMDC.setLocation(subject.getId().toString()); - subject.setAuthenticationInfo(info); - return Optional.of(subject); - } - -} +package com.bakdata.conquery.models.auth; + +import java.util.Optional; + +import com.bakdata.conquery.models.auth.entities.Subject; +import com.bakdata.conquery.models.auth.web.AuthenticationExceptionMapper; +import com.bakdata.conquery.util.io.ConqueryMDC; +import io.dropwizard.auth.Authenticator; +import lombok.extern.slf4j.Slf4j; +import org.apache.shiro.SecurityUtils; +import org.apache.shiro.authc.AuthenticationToken; + +/** + * This dropwizard authenticator and the shiro realm are conceptually the same. + * They authenticate -- but shiro realms can also be used for authorization. + * We use this single authenticator to set up shiro and forward all requests to + * shiro, where multiple realms might be configured. + * We need this authenticator to plug in the security, and hereby shiro, into the AuthFilter. + */ +@Slf4j +public class ConqueryAuthenticator implements Authenticator { + + /** + * The execeptions thrown by Shiro will be catched by {@link AuthenticationExceptionMapper}. + */ + @Override + public Optional authenticate(AuthenticationToken token) { + // Submit the token to Shiro (to all realms that were registered) + ConqueryAuthenticationInfo info = (ConqueryAuthenticationInfo) SecurityUtils.getSecurityManager() + .authenticate( + token); + + // Extract + Subject subject = info.getPrincipals().oneByType(Subject.class); + + + // If the subject was present, all further authorization can now be performed on the subject object + log.trace("Using subject {} for further authorization", subject); + ConqueryMDC.setLocation(subject.getId().toString()); + subject.setAuthenticationInfo(info); + return Optional.of(subject); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/ConqueryAuthorizationRealm.java b/backend/src/main/java/com/bakdata/conquery/models/auth/ConqueryAuthorizationRealm.java index de2fd080102..fc90cde8c27 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/ConqueryAuthorizationRealm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/ConqueryAuthorizationRealm.java @@ -1,137 +1,143 @@ -package com.bakdata.conquery.models.auth; - -import java.util.Collection; -import java.util.Collections; -import java.util.ConcurrentModificationException; -import java.util.Objects; -import java.util.Set; - -import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.models.auth.entities.Subject; -import com.bakdata.conquery.models.identifiable.ids.specific.UserId; -import com.google.common.collect.Sets; -import lombok.RequiredArgsConstructor; -import org.apache.shiro.authc.AuthenticationException; -import org.apache.shiro.authc.AuthenticationInfo; -import org.apache.shiro.authc.AuthenticationToken; -import org.apache.shiro.authz.AuthorizationInfo; -import org.apache.shiro.authz.Permission; -import org.apache.shiro.authz.SimpleAuthorizationInfo; -import org.apache.shiro.realm.AuthorizingRealm; -import org.apache.shiro.subject.PrincipalCollection; - -/** - * This realms only provides authorization information for a given {@link UserId}. - * For now there is only one such authorizing realm. This queries the {@link MetaStorage}. - */ -@RequiredArgsConstructor -public class ConqueryAuthorizationRealm extends AuthorizingRealm { - - public final MetaStorage storage; - - @Override - protected void onInit() { - super.onInit(); - /* - * We don't handle authentication here, thus no token is supported. However we - * need to provide a TokenClass (that is used nowhere else), to not cause a - * NullPointerException (see AuthenticatingRealm#supports). - */ - this.setAuthenticationTokenClass(UnusedAuthenticationToken.class); - } - - @Override - protected AuthorizationInfo doGetAuthorizationInfo(PrincipalCollection principals) { - Objects.requireNonNull(principals, "No principal info was provided"); - Subject subject = principals.oneByType(Subject.class); - SimpleAuthorizationInfo info = new ConqueryAuthorizationInfo(); - - info.addObjectPermissions(Collections.unmodifiableSet(subject.getUser().getEffectivePermissions())); - - return info; - } - - @Override - protected AuthenticationInfo doGetAuthenticationInfo(AuthenticationToken token) throws AuthenticationException { - // This Realm only authorizes - return null; - } - - /** - * Dummy class for the TokenClass that is signals that this realm does not authenticate. - */ - @SuppressWarnings("serial") - private static class UnusedAuthenticationToken implements AuthenticationToken { - - @Override - public Object getPrincipal() { - throw new UnsupportedOperationException(String.format("This realm (%s) only handles authorization. So this token's functions should never be called.", this.getClass().getName())); - } - - @Override - public Object getCredentials() { - throw new UnsupportedOperationException(String.format("This realm (%s) only handles authorization. So this token's functions should never be called.", this.getClass().getName())); - } - - } - - /** - * This AuthorizationInfo handles the collection of large amounts of {@link Permission}s by wrapping collections into a view - * instead of running an iterator over them. This also prevents a {@link ConcurrentModificationException} which occurred when - * Permission were collected - */ - @SuppressWarnings("serial") - public static class ConqueryAuthorizationInfo extends SimpleAuthorizationInfo { - @Override - public void addRole(String role) { - throw new UnsupportedOperationException(); - } - - @Override - public void addRoles(Collection roles) { - throw new UnsupportedOperationException(); - } - - @Override - public void addStringPermission(String permission) { - throw new UnsupportedOperationException(); - } - - @Override - public void addStringPermissions(Collection permissions) { - throw new UnsupportedOperationException(); - } - - @Override - public void addObjectPermission(Permission permission) { - throw new UnsupportedOperationException(); - } - - @Override - public void addObjectPermissions(Collection permissions) { - if (!(permissions instanceof Set)) { - super.addObjectPermissions(permissions); - } - if (objectPermissions == null) { - objectPermissions = (Set) permissions; - return; - } - objectPermissions = Sets.union(objectPermissions, (Set)permissions); - } - - } - - // We override this to work only on SetViews and allocate new sets - @Override - protected Collection getPermissions(AuthorizationInfo info) { - // The AuthorizationInfo must be a ConqueryAuthorizationInfo and should only hold ObjectPermissions - Collection perms = info.getObjectPermissions(); - - if (perms.isEmpty()) { - return Collections.emptySet(); - } else { - return perms; - } - } - -} +package com.bakdata.conquery.models.auth; + +import java.util.Collection; +import java.util.Collections; +import java.util.ConcurrentModificationException; +import java.util.Objects; +import java.util.Set; + +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.models.auth.entities.Subject; +import com.bakdata.conquery.models.identifiable.ids.specific.UserId; +import com.google.common.collect.Sets; +import lombok.RequiredArgsConstructor; +import org.apache.shiro.authc.AuthenticationException; +import org.apache.shiro.authc.AuthenticationInfo; +import org.apache.shiro.authc.AuthenticationToken; +import org.apache.shiro.authz.AuthorizationInfo; +import org.apache.shiro.authz.Permission; +import org.apache.shiro.authz.SimpleAuthorizationInfo; +import org.apache.shiro.realm.AuthorizingRealm; +import org.apache.shiro.subject.PrincipalCollection; + +/** + * This realms only provides authorization information for a given {@link UserId}. + * For now there is only one such authorizing realm. This queries the {@link MetaStorage}. + */ +@RequiredArgsConstructor +public class ConqueryAuthorizationRealm extends AuthorizingRealm { + + public final MetaStorage storage; + + @Override + protected void onInit() { + super.onInit(); + /* + * We don't handle authentication here, thus no token is supported. However we + * need to provide a TokenClass (that is used nowhere else), to not cause a + * NullPointerException (see AuthenticatingRealm#supports). + */ + this.setAuthenticationTokenClass(UnusedAuthenticationToken.class); + } + + @Override + protected AuthorizationInfo doGetAuthorizationInfo(PrincipalCollection principals) { + Objects.requireNonNull(principals, "No principal info was provided"); + Subject subject = principals.oneByType(Subject.class); + SimpleAuthorizationInfo info = new ConqueryAuthorizationInfo(); + + info.addObjectPermissions(Collections.unmodifiableSet(subject.getUser().getEffectivePermissions())); + + return info; + } + + @Override + protected AuthenticationInfo doGetAuthenticationInfo(AuthenticationToken token) throws AuthenticationException { + // This Realm only authorizes + return null; + } + + /** + * Dummy class for the TokenClass that is signals that this realm does not authenticate. + */ + @SuppressWarnings("serial") + private static class UnusedAuthenticationToken implements AuthenticationToken { + + @Override + public Object getPrincipal() { + throw new UnsupportedOperationException( + String.format( + "This realm (%s) only handles authorization. So this token's functions should never be called.", + this.getClass().getName())); + } + + @Override + public Object getCredentials() { + throw new UnsupportedOperationException( + String.format( + "This realm (%s) only handles authorization. So this token's functions should never be called.", + this.getClass().getName())); + } + + } + + /** + * This AuthorizationInfo handles the collection of large amounts of {@link Permission}s by wrapping collections into a view + * instead of running an iterator over them. This also prevents a {@link ConcurrentModificationException} which occurred when + * Permission were collected + */ + @SuppressWarnings("serial") + public static class ConqueryAuthorizationInfo extends SimpleAuthorizationInfo { + @Override + public void addRole(String role) { + throw new UnsupportedOperationException(); + } + + @Override + public void addRoles(Collection roles) { + throw new UnsupportedOperationException(); + } + + @Override + public void addStringPermission(String permission) { + throw new UnsupportedOperationException(); + } + + @Override + public void addStringPermissions(Collection permissions) { + throw new UnsupportedOperationException(); + } + + @Override + public void addObjectPermission(Permission permission) { + throw new UnsupportedOperationException(); + } + + @Override + public void addObjectPermissions(Collection permissions) { + if (!(permissions instanceof Set)) { + super.addObjectPermissions(permissions); + } + if (objectPermissions == null) { + objectPermissions = (Set) permissions; + return; + } + objectPermissions = Sets.union(objectPermissions, (Set) permissions); + } + + } + + // We override this to work only on SetViews and allocate new sets + @Override + protected Collection getPermissions(AuthorizationInfo info) { + // The AuthorizationInfo must be a ConqueryAuthorizationInfo and should only hold ObjectPermissions + Collection perms = info.getObjectPermissions(); + + if (perms.isEmpty()) { + return Collections.emptySet(); + } else { + return perms; + } + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/UserManageable.java b/backend/src/main/java/com/bakdata/conquery/models/auth/UserManageable.java index 96dac2976c5..bc503c7126a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/UserManageable.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/UserManageable.java @@ -1,41 +1,41 @@ -package com.bakdata.conquery.models.auth; - -import java.util.List; - -import com.bakdata.conquery.Conquery; -import com.bakdata.conquery.apiv1.auth.CredentialType; -import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.models.auth.entities.User; -import com.bakdata.conquery.models.identifiable.ids.specific.UserId; - -/** - * Basic functionality for a realm to manage users and interface with {@link Conquery}. - */ -public interface UserManageable { - - /** - * Add a user to an authenticating realm. The realm is responsible for picking the appropriate {@link CredentialType} from the argument. - * If not fitting type was found the user should not be added. - * - * @param credentials A List of credentials that are provided by the user. - * @param userId - * @return True upon successful adding of the user. False if the user could not be added or was already present. - */ - boolean addUser(UserId userId, CredentialType credential); - - - /** - * Similar to {@link UserManageable#addUser(User, List)} but if the user already existed it is overridden, when a fitting {@link CredentialType} was found. - */ - boolean updateUser(UserId user, CredentialType credential); - - /** - * Removes a user from the realm only but not from the local permission storage (i.e. {@link MetaStorage}). - */ - boolean removeUser(UserId user); - - /** - * Returns a list of all users managed by the authenticating realm as {@link UserId}s. - */ - List getAllUsers(); -} +package com.bakdata.conquery.models.auth; + +import java.util.List; + +import com.bakdata.conquery.Conquery; +import com.bakdata.conquery.apiv1.auth.CredentialType; +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.models.auth.entities.User; +import com.bakdata.conquery.models.identifiable.ids.specific.UserId; + +/** + * Basic functionality for a realm to manage users and interface with {@link Conquery}. + */ +public interface UserManageable { + + /** + * Add a user to an authenticating realm. The realm is responsible for picking the appropriate {@link CredentialType} from the argument. + * If not fitting type was found the user should not be added. + * + * @param credential A credentials that is provided by the user. + * @param userId + * @return True upon successful adding of the user. False if the user could not be added or was already present. + */ + boolean addUser(UserId userId, CredentialType credential); + + + /** + * Similar to {@link UserManageable#addUser(User, List)} but if the user already existed it is overridden, when a fitting {@link CredentialType} was found. + */ + boolean updateUser(UserId user, CredentialType credential); + + /** + * Removes a user from the realm only but not from the local permission storage (i.e. {@link MetaStorage}). + */ + boolean removeUser(UserId user); + + /** + * Returns a list of all users managed by the authenticating realm as {@link UserId}s. + */ + List getAllUsers(); +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/basic/AccessTokenCreator.java b/backend/src/main/java/com/bakdata/conquery/models/auth/basic/AccessTokenCreator.java index b5dceb31942..1da5bfe88f7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/basic/AccessTokenCreator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/basic/AccessTokenCreator.java @@ -9,4 +9,4 @@ public interface AccessTokenCreator { */ String createAccessToken(String username, String password); -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/basic/JWTokenHandler.java b/backend/src/main/java/com/bakdata/conquery/models/auth/basic/JWTokenHandler.java index 04930ca2012..bddf4af837b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/basic/JWTokenHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/basic/JWTokenHandler.java @@ -1,9 +1,10 @@ package com.bakdata.conquery.models.auth.basic; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.core.HttpHeaders; import java.security.SecureRandom; import java.util.Date; import java.util.Random; - import javax.annotation.Nullable; import com.auth0.jwt.JWT; @@ -12,8 +13,6 @@ import com.bakdata.conquery.models.auth.web.AuthFilter; import io.dropwizard.auth.oauth.OAuthCredentialAuthFilter; import io.dropwizard.util.Duration; -import jakarta.ws.rs.container.ContainerRequestContext; -import jakarta.ws.rs.core.HttpHeaders; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; @@ -38,7 +37,13 @@ public class JWTokenHandler { public String createToken(String username, Duration duration, String issuer, Algorithm algorithm) { Date issueDate = new Date(); Date expDate = DateUtils.addMinutes(issueDate, Long.valueOf(duration.toMinutes()).intValue()); - String token = JWT.create().withIssuer(issuer).withSubject(username).withIssuedAt(issueDate).withExpiresAt(expDate).sign(algorithm); + String token = JWT.create() + .withIssuer(issuer) + .withSubject(username) + .withIssuedAt(issueDate) + .withExpiresAt( + expDate) + .sign(algorithm); return token; } @@ -108,16 +113,14 @@ public AuthenticationToken apply(ContainerRequestContext request) { if (tokenHeader == null && tokenQuery == null) { // No token could be parsed return null; - } - else if (tokenHeader != null && tokenQuery != null) { - log.warn("There were tokens in the request header and query string provided, which is forbidden. See: https://tools.ietf.org/html/rfc6750#section-2"); + } else if (tokenHeader != null && tokenQuery != null) { + log.warn( + "There were tokens in the request header and query string provided, which is forbidden. See: https://tools.ietf.org/html/rfc6750#section-2"); return null; - } - else if (tokenHeader != null) { + } else if (tokenHeader != null) { log.trace("Extracted the request header token"); token = tokenHeader; - } - else { + } else { log.trace("Extracted the query string token"); token = tokenQuery; } @@ -126,8 +129,7 @@ else if (tokenHeader != null) { JWT.decode(token); return new BearerToken(token); - } - catch (JWTDecodeException e) { + } catch (JWTDecodeException e) { return null; } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/basic/LocalAuthenticationRealm.java b/backend/src/main/java/com/bakdata/conquery/models/auth/basic/LocalAuthenticationRealm.java index 7f48c4b0fca..dd8e0ecf16b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/basic/LocalAuthenticationRealm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/basic/LocalAuthenticationRealm.java @@ -1,10 +1,10 @@ package com.bakdata.conquery.models.auth.basic; +import jakarta.validation.Validator; import java.io.File; import java.io.IOException; import java.util.List; import java.util.concurrent.Executors; -import jakarta.validation.Validator; import com.bakdata.conquery.Conquery; import com.bakdata.conquery.apiv1.auth.CredentialType; @@ -79,8 +79,17 @@ public class LocalAuthenticationRealm extends AuthenticatingRealm implements Con //////////////////// INITIALIZATION //////////////////// - public LocalAuthenticationRealm(Validator validator, ObjectMapper mapper, ConqueryTokenRealm centralTokenRealm, String storeName, File storageDir, XodusConfig passwordStoreConfig, Duration validDuration, HashingFunction defaultHashingFunction, - CaffeineSpec caffeineSpec, MetricRegistry metricRegistry) { + public LocalAuthenticationRealm( + Validator validator, + ObjectMapper mapper, + ConqueryTokenRealm centralTokenRealm, + String storeName, + File storageDir, + XodusConfig passwordStoreConfig, + Duration validDuration, + HashingFunction defaultHashingFunction, + CaffeineSpec caffeineSpec, + MetricRegistry metricRegistry) { this.validator = validator; this.mapper = mapper; this.defaultHashingFunction = defaultHashingFunction; @@ -101,24 +110,25 @@ protected void onInit() { File passwordStoreFile = new File(storageDir, storeName); passwordEnvironment = Environments.newInstance(passwordStoreFile, passwordStoreConfig.createConfig()); passwordStore = new CachedStore<>( - new SerializingStore<>( - new XodusStore( - passwordEnvironment, - "passwords", - store -> store.getEnvironment().close(), - store -> { - } - ), - validator, - mapper, - UserId.class, - HashEntry.class, - false, - true, - null, Executors.newSingleThreadExecutor() + new SerializingStore<>( + new XodusStore( + passwordEnvironment, + "passwords", + store -> store.getEnvironment().close(), + store -> { + } ), - caffeineSpec, - metricRegistry + validator, + mapper, + UserId.class, + HashEntry.class, + false, + true, + null, + Executors.newSingleThreadExecutor() + ), + caffeineSpec, + metricRegistry ); } @@ -129,8 +139,10 @@ protected void onInit() { * Should not be called since the tokens are now handled by the ConqueryTokenRealm. */ @Override - public ConqueryAuthenticationInfo doGetAuthenticationInfo(AuthenticationToken token) throws AuthenticationException { - throw new UnsupportedOperationException("Should not be called since the tokens are now handled by the ConqueryTokenRealm."); + public ConqueryAuthenticationInfo doGetAuthenticationInfo( + AuthenticationToken token) throws AuthenticationException { + throw new UnsupportedOperationException( + "Should not be called since the tokens are now handled by the ConqueryTokenRealm."); } //////////////////// FOR USERNAME/PASSWORD @@ -164,14 +176,13 @@ public boolean addUser(UserId user, @NonNull CredentialType credential) { passwordStore.add(user, hashEntry); log.debug("Added user to realm: {}", user); return true; - } - catch (IllegalArgumentException e) { + } catch (IllegalArgumentException e) { log.warn("Unable to add user '{}'", user, e); } return false; } - //////////////////// USER MANAGEMENT //////////////////// + //////////////////// USER MANAGEMENT /** * Converts the provided password to a Xodus compatible hash. @@ -180,11 +191,8 @@ private HashEntry toHashEntry(CredentialType credential) { if (credential instanceof PasswordCredential(String password)) { - return new HashEntry(Password.hash(password) - .with(defaultHashingFunction) - .getResult()); - } - else if (credential instanceof PasswordHashCredential(String hash)) { + return new HashEntry(Password.hash(password).with(defaultHashingFunction).getResult()); + } else if (credential instanceof PasswordHashCredential(String hash)) { return new HashEntry(hash); } @@ -203,8 +211,7 @@ public boolean updateUser(UserId user, CredentialType credential) { final HashEntry hashEntry = toHashEntry(credential); passwordStore.update(user, hashEntry); return true; - } - catch (IllegalArgumentException e) { + } catch (IllegalArgumentException e) { log.warn("Unable to update user '{}'", user, e); } return false; @@ -223,25 +230,23 @@ public List getAllUsers() { } - - //////////////////// LIFECYCLE MANAGEMENT //////////////////// - + //////////////////// LIFECYCLE MANAGEMENT + @Override @SneakyThrows(IOException.class) public void destroy() throws InterruptedException { - for(int retries = 0; retries < ENVIRONMENT_CLOSING_RETRIES; retries++) { + for (int retries = 0; retries < ENVIRONMENT_CLOSING_RETRIES; retries++) { try { log.info("Closing the password environment."); passwordStore.close(); return; - } - catch (EnvironmentClosedException e) { - log.warn("Password environment was already closed, which is odd but maybe the stop() lifecycle event fired twice"); + } catch (EnvironmentClosedException e) { + log.warn( + "Password environment was already closed, which is odd but maybe the stop() lifecycle event fired twice"); return; - } - catch (ExodusException e) { + } catch (ExodusException e) { if (retries == 0) { - log.info("The environment is still working on some transactions. Retry"); + log.info("The environment is still working on some transactions. Retry"); } log.info("Waiting for {} seconds to retry.", ENVIRONMENT_CLOSING_TIMEOUT); diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/basic/PasswordHelper.java b/backend/src/main/java/com/bakdata/conquery/models/auth/basic/PasswordHelper.java index 549c0ee70fb..2689eb24950 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/basic/PasswordHelper.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/basic/PasswordHelper.java @@ -15,26 +15,34 @@ @Slf4j public class PasswordHelper { - private final static Map, Function> HASH_FUNCTION_GENERATORS = Map.of( - Argon2Function.class, Argon2Function::getInstanceFromHash, - ScryptFunction.class, ScryptFunction::getInstanceFromHash, - BcryptFunction.class, BcryptFunction::getInstanceFromHash, - CompressedPBKDF2Function.class, CompressedPBKDF2Function::getInstanceFromHash - ); + private final static Map, Function> HASH_FUNCTION_GENERATORS = Map + .of( + Argon2Function.class, + Argon2Function::getInstanceFromHash, + ScryptFunction.class, + ScryptFunction::getInstanceFromHash, + BcryptFunction.class, + BcryptFunction::getInstanceFromHash, + CompressedPBKDF2Function.class, + CompressedPBKDF2Function::getInstanceFromHash + ); /** * Determines the function used to create the provided hash. * The function can be used to hash a plain credential in order to check the hashes for equality. */ public HashingFunction getHashingFunction(String hash) { - for (Map.Entry, Function> hashFunctionGenerator : HASH_FUNCTION_GENERATORS.entrySet()) { + for (Map.Entry, Function> hashFunctionGenerator : HASH_FUNCTION_GENERATORS + .entrySet()) { try { return hashFunctionGenerator.getValue().apply(hash); - } - catch (Exception e) { - log.trace("Could not create hash function instance from hash using '{}'", hashFunctionGenerator.getKey()); + } catch (Exception e) { + log.trace( + "Could not create hash function instance from hash using '{}'", + hashFunctionGenerator.getKey()); } } - throw new IllegalArgumentException("No supported hash function recognized hash. Supported functions: " + HASH_FUNCTION_GENERATORS.keySet()); + throw new IllegalArgumentException( + "No supported hash function recognized hash. Supported functions: " + HASH_FUNCTION_GENERATORS.keySet()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/basic/UserAuthenticationManagementProcessor.java b/backend/src/main/java/com/bakdata/conquery/models/auth/basic/UserAuthenticationManagementProcessor.java index 0acf3fe91e0..6df5d367d35 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/basic/UserAuthenticationManagementProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/basic/UserAuthenticationManagementProcessor.java @@ -1,48 +1,48 @@ -package com.bakdata.conquery.models.auth.basic; - -import com.bakdata.conquery.apiv1.auth.ProtoUser; -import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.models.auth.AuthorizationHelper; -import com.bakdata.conquery.models.auth.entities.User; -import com.bakdata.conquery.models.identifiable.ids.specific.UserId; -import com.bakdata.conquery.resources.admin.rest.UserAuthenticationManagementResource; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -/** - * Business logic for the {@link UserAuthenticationManagementResource}. - */ -@RequiredArgsConstructor -@Slf4j -public class UserAuthenticationManagementProcessor { - - private final LocalAuthenticationRealm realm; - private final MetaStorage storage; - - public boolean tryRegister(ProtoUser pUser) { - final UserId id = pUser.createId(); - User user = storage.getUser(id); - if (user == null) { - log.warn("Unable to add new user {}. Probably already existed.", pUser); - return false; - } - log.trace("Added the user {} to the authorization storage", id); - if (AuthorizationHelper.registerForAuthentication(realm, user, pUser.getCredential(), false)) { - log.trace("Added the user {} to the realm {}", id, realm.getName()); - return true; - } - log.trace("Failed to add added the user {} to the realm {}", id, realm.getName()); - return false; - } - - public boolean updateUser(ProtoUser pUser) { - final User user = pUser.createOrOverwriteUser(storage); - AuthorizationHelper.registerForAuthentication(realm, user, pUser.getCredential(), false); - return true; - } - - public void remove(UserId user) { - realm.removeUser(user); - } - -} +package com.bakdata.conquery.models.auth.basic; + +import com.bakdata.conquery.apiv1.auth.ProtoUser; +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.models.auth.AuthorizationHelper; +import com.bakdata.conquery.models.auth.entities.User; +import com.bakdata.conquery.models.identifiable.ids.specific.UserId; +import com.bakdata.conquery.resources.admin.rest.UserAuthenticationManagementResource; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +/** + * Business logic for the {@link UserAuthenticationManagementResource}. + */ +@RequiredArgsConstructor +@Slf4j +public class UserAuthenticationManagementProcessor { + + private final LocalAuthenticationRealm realm; + private final MetaStorage storage; + + public boolean tryRegister(ProtoUser pUser) { + final UserId id = pUser.createId(); + User user = storage.getUser(id); + if (user == null) { + log.warn("Unable to add new user {}. Probably already existed.", pUser); + return false; + } + log.trace("Added the user {} to the authorization storage", id); + if (AuthorizationHelper.registerForAuthentication(realm, user, pUser.getCredential(), false)) { + log.trace("Added the user {} to the realm {}", id, realm.getName()); + return true; + } + log.trace("Failed to add added the user {} to the realm {}", id, realm.getName()); + return false; + } + + public boolean updateUser(ProtoUser pUser) { + final User user = pUser.createOrOverwriteUser(storage); + AuthorizationHelper.registerForAuthentication(realm, user, pUser.getCredential(), false); + return true; + } + + public void remove(UserId user) { + realm.removeUser(user); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/conquerytoken/ConqueryTokenRealm.java b/backend/src/main/java/com/bakdata/conquery/models/auth/conquerytoken/ConqueryTokenRealm.java index d808260d759..f5de3063acb 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/conquerytoken/ConqueryTokenRealm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/conquerytoken/ConqueryTokenRealm.java @@ -38,8 +38,8 @@ public class ConqueryTokenRealm extends AuthenticatingRealm implements ConqueryA private final MetaStorage storage; @Setter private JWTConfig jwtConfig = new JWTConfig(); - - + + public ConqueryTokenRealm(MetaStorage storage) { this.storage = storage; setAuthenticationTokenClass(TOKEN_CLASS); @@ -47,7 +47,8 @@ public ConqueryTokenRealm(MetaStorage storage) { } @Override - public ConqueryAuthenticationInfo doGetAuthenticationInfo(AuthenticationToken token) throws AuthenticationException { + public ConqueryAuthenticationInfo doGetAuthenticationInfo( + AuthenticationToken token) throws AuthenticationException { if (!(TOKEN_CLASS.isAssignableFrom(token.getClass()))) { log.trace("Incompatible token. Expected {}, got {}", TOKEN_CLASS, token.getClass()); return null; @@ -56,20 +57,16 @@ public ConqueryAuthenticationInfo doGetAuthenticationInfo(AuthenticationToken to DecodedJWT decodedToken = null; try { decodedToken = jwtConfig.getTokenVerifier(this).verify((String) token.getCredentials()); - } - catch (TokenExpiredException e) { + } catch (TokenExpiredException e) { log.trace("The provided token is expired."); throw new ExpiredCredentialsException(e); - } - catch (SignatureVerificationException | InvalidClaimException e) { + } catch (SignatureVerificationException | InvalidClaimException e) { log.trace("The provided token was not successfully verified against its signature or claims."); throw new IncorrectCredentialsException(e); - } - catch (JWTVerificationException e) { + } catch (JWTVerificationException e) { log.trace("The provided token could not be verified.", e); throw new AuthenticationException(e); - } - catch (Exception e) { + } catch (Exception e) { log.trace("Unable to decode token", e); throw new AuthenticationException(e); } @@ -86,37 +83,41 @@ public ConqueryAuthenticationInfo doGetAuthenticationInfo(AuthenticationToken to public String createTokenForUser(@NonNull UserId userId, @NonNull Duration validDuration) { - if(storage.getUser(userId) == null) { + if (storage.getUser(userId) == null) { throw new IllegalArgumentException("Cannot create a JWT for unknown user with id: " + userId); } - return JWTokenHandler.createToken(userId.toString(), validDuration, getName(), jwtConfig.getTokenSignAlgorithm()); + return JWTokenHandler.createToken( + userId.toString(), + validDuration, + getName(), + jwtConfig.getTokenSignAlgorithm()); } - + public String createTokenForUser(UserId userId) { return createTokenForUser(userId, jwtConfig.getJwtDuration()); } - - public static class JWTConfig{ + + public static class JWTConfig { @Getter @Setter private Duration jwtDuration = Duration.hours(8); - + @JsonIgnore @Getter private Algorithm tokenSignAlgorithm = Algorithm.HMAC256(JWTokenHandler.generateTokenSecret()); @JsonIgnore private JWTVerifier tokenVerifier; - + @JsonIgnore public JWTVerifier getTokenVerifier(AuthenticatingRealm realm) { - if(tokenVerifier == null) { + if (tokenVerifier == null) { tokenVerifier = JWT.require(tokenSignAlgorithm).withIssuer(realm.getName()).build(); } return tokenVerifier; } - + } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevAuthConfig.java b/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevAuthConfig.java index d28e350c440..2e406fd85f6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevAuthConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevAuthConfig.java @@ -15,14 +15,21 @@ */ @CPSType(base = AuthenticationRealmFactory.class, id = "DEVELOPMENT") public class DevAuthConfig implements AuthenticationRealmFactory { - + @Override - public ConqueryAuthenticationRealm createRealm(Environment environment, ConqueryConfig config, AuthorizationController authorizationController) { + public ConqueryAuthenticationRealm createRealm( + Environment environment, + ConqueryConfig config, + AuthorizationController authorizationController) { AuthFilter.registerTokenExtractor(UserIdTokenExtractor.class, environment.jersey().getResourceConfig()); if (authorizationController.getAdminServlet() != null) { - AuthFilter.registerTokenExtractor(UserIdTokenExtractor.class, authorizationController.getAdminServlet().getJerseyConfig()); - AuthFilter.registerTokenExtractor(UserIdTokenExtractor.class, authorizationController.getAdminServlet().getJerseyConfigUI()); + AuthFilter.registerTokenExtractor( + UserIdTokenExtractor.class, + authorizationController.getAdminServlet().getJerseyConfig()); + AuthFilter.registerTokenExtractor( + UserIdTokenExtractor.class, + authorizationController.getAdminServlet().getJerseyConfigUI()); } // Use the first defined user als the default user. This is usually the superuser if the DevelopmentAuthorizationConfig is set diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevelopmentToken.java b/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevelopmentToken.java index 9d98a122d18..faade85f086 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevelopmentToken.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevelopmentToken.java @@ -5,12 +5,10 @@ import lombok.RequiredArgsConstructor; import org.apache.shiro.authc.AuthenticationToken; -@SuppressWarnings("serial") @Getter @RequiredArgsConstructor public class DevelopmentToken implements AuthenticationToken { - //TODO migrate to user? private final UserId principal; private final String credentials; -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/develop/FirstInitialUserRealm.java b/backend/src/main/java/com/bakdata/conquery/models/auth/develop/FirstInitialUserRealm.java index 924a6d8acf2..985ff245844 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/develop/FirstInitialUserRealm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/develop/FirstInitialUserRealm.java @@ -19,7 +19,7 @@ * identifying information was found in the request. If the realm was able to * parse a {@link UserId} from a request, it submits this id in an * {@link AuthenticationToken}. - * + * */ @Slf4j @RequiredArgsConstructor @@ -33,19 +33,7 @@ public class FirstInitialUserRealm extends AuthenticatingRealm implements Conque /** * The warning that is displayed, when the realm is instantiated. */ - private static final String WARNING = "\n" - + " §§\n" - + " § §\n" - + " § §\n" - + " § §\n" - + " § §§§§ § You instantiated and are probably using a Shiro realm\n" - + " § §§§§ § that does not do any permission checks or authentication.\n" - + " § §§ § Access to all resources is granted to everyone.\n" - + " § §§ § DO NOT USE THIS REALM IN PRODUCTION\n" - + " $ §\n" - + " § §§ §\n" - + " § §\n" - + " §§§§§§§§§§§§§§§§§§§§§§"; + private static final String WARNING = "\n" + " §§\n" + " § §\n" + " § §\n" + " § §\n" + " § §§§§ § You instantiated and are probably using a Shiro realm\n" + " § §§§§ § that does not do any permission checks or authentication.\n" + " § §§ § Access to all resources is granted to everyone.\n" + " § §§ § DO NOT USE THIS REALM IN PRODUCTION\n" + " $ §\n" + " § §§ §\n" + " § §\n" + " §§§§§§§§§§§§§§§§§§§§§§"; static { log.warn(WARNING); @@ -59,7 +47,8 @@ protected void onInit() { } @Override - public ConqueryAuthenticationInfo doGetAuthenticationInfo(AuthenticationToken token) throws AuthenticationException { + public ConqueryAuthenticationInfo doGetAuthenticationInfo( + AuthenticationToken token) throws AuthenticationException { if (!(token instanceof DevelopmentToken devToken)) { return null; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/develop/UserIdTokenExtractor.java b/backend/src/main/java/com/bakdata/conquery/models/auth/develop/UserIdTokenExtractor.java index cd906eac6ca..0f2fa2eceb8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/develop/UserIdTokenExtractor.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/develop/UserIdTokenExtractor.java @@ -1,10 +1,11 @@ package com.bakdata.conquery.models.auth.develop; -import com.bakdata.conquery.models.auth.web.AuthFilter; -import com.bakdata.conquery.models.identifiable.ids.specific.UserId; import jakarta.inject.Named; import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.HttpHeaders; + +import com.bakdata.conquery.models.auth.web.AuthFilter; +import com.bakdata.conquery.models.identifiable.ids.specific.UserId; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -32,8 +33,7 @@ public AuthenticationToken apply(ContainerRequestContext requestContext) { if (uid != null) { uid = uid.replaceFirst("^Bearer ", ""); - } - else { + } else { // Check also the query parameter "access_token" for a UserId uid = requestContext.getUriInfo().getQueryParameters().getFirst(UID_QUERY_STRING_PARAMETER); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/FilteredUser.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/FilteredUser.java index 35ba82cab81..194de9d04e5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/FilteredUser.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/FilteredUser.java @@ -16,9 +16,9 @@ /** * Class to filter all unused functionality from the actual User class to keep it more compact and clean. * These are not supposed to be implemented. - * + * */ -@JsonIgnoreProperties({ "session", "previousPrincipals", "runAs", "principal", "authenticated", "remembered", "principals" }) +@JsonIgnoreProperties({"session", "previousPrincipals", "runAs", "principal", "authenticated", "remembered", "principals"}) public abstract class FilteredUser implements Subject { @@ -146,6 +146,6 @@ public boolean isAuthenticated() { public boolean isRemembered() { throw new UnsupportedOperationException(); } - + } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java index 6d3871ce942..750df141469 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java @@ -1,163 +1,158 @@ -package com.bakdata.conquery.models.auth.entities; - -import java.util.Collections; -import java.util.Comparator; -import java.util.HashSet; -import java.util.Set; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; - -import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; -import com.bakdata.conquery.models.identifiable.MetaIdentifiable; -import com.bakdata.conquery.models.identifiable.ids.specific.PermissionOwnerId; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.google.common.collect.ImmutableSet; -import lombok.AccessLevel; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.NonNull; -import lombok.Setter; -import lombok.ToString; -import lombok.extern.slf4j.Slf4j; -import org.apache.shiro.authz.Permission; - -/** - * The base class of security subjects in this project. Used to represent - * persons and groups with permissions. - * - * @param The id type by which an instance is identified. - * @implNote The NoArgsConstructor is private and used for deserialization - */ -@Slf4j -@EqualsAndHashCode(callSuper = false) -@NoArgsConstructor(access = AccessLevel.PROTECTED) -public abstract class PermissionOwner>> extends MetaIdentifiable implements Comparable> { - - private static final Comparator> - COMPARATOR = - Comparator., String>comparing(PermissionOwner::getLabel).thenComparing(po -> po.getId().toString()); - - @Getter - @Setter - @NonNull - @NotNull - @NotEmpty - @ToString.Include - protected String name; - - @Getter - @Setter - @NonNull - @NotNull - @NotEmpty - @ToString.Include - protected String label; - - @ToString.Exclude - @NotNull - private Set permissions = new HashSet<>(); - - - @JsonCreator - public PermissionOwner(String name, String label) { - this.name = name; - this.label = label; - } - - public PermissionOwner(String name, String label, MetaStorage storage) { - this.name = name; - this.label = label; - setMetaStorage(storage); - } - - - /** - * Adds permissions to the owner object and to the persistent storage. - * - * @param permissions The permissions to add. - * @return Returns the added Permission - */ - public synchronized void addPermissions(Set permissions) { - this.permissions = ImmutableSet - .builder() - .addAll(this.permissions) - .addAll(permissions) - .build(); - updateStorage(); - } - - public synchronized void addPermission(ConqueryPermission permission) { - this.permissions = ImmutableSet - .builder() - .addAll(this.permissions) - .add(permission) - .build(); - updateStorage(); - } - - public boolean removePermission(Permission permission) { - boolean ret = false; - synchronized (this) { - Set newSet = new HashSet<>(this.permissions); - ret = newSet.remove(permission); - this.permissions = newSet; - updateStorage(); - } - return ret; - } - - /** - * Returns a copy of the permissions hold by the owner. - * Changes to the returned collection are not persisted and do not alter the - * permissions owned. - * - * @return A set of the permissions hold by the owner. - */ - public Set getPermissions() { - if (permissions.isEmpty()) { - return Collections.emptySet(); - } - return Collections.unmodifiableSet(permissions); - - } - - /** - * Returns a collection of the effective permissions. These are the permissions of the owner and - * the permission of the roles/groups it inherits from. - * - * @return Owned and inherited permissions. - */ - @JsonIgnore - public abstract Set getEffectivePermissions(); - - public void updatePermissions(Set permissionsNew) { - synchronized (this) { - Set newSet = new HashSet<>(permissionsNew.size()); - newSet.addAll(permissionsNew); - this.permissions = newSet; - updateStorage(); - } - } - - /** - * Update this instance in the {@link MetaStorage}. - */ - protected abstract void updateStorage(); - - - @Override - public int compareTo(PermissionOwner other) { - return COMPARATOR.compare(this, other); - } - - @Override - public abstract T createId(); - - @Override - public T getId() { - return super.getId(); - } -} +package com.bakdata.conquery.models.auth.entities; + +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.Set; + +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; +import com.bakdata.conquery.models.identifiable.MetaIdentifiable; +import com.bakdata.conquery.models.identifiable.ids.specific.PermissionOwnerId; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.collect.ImmutableSet; +import lombok.AccessLevel; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import lombok.Setter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; +import org.apache.shiro.authz.Permission; + +/** + * The base class of security subjects in this project. Used to represent + * persons and groups with permissions. + * + * @param The id type by which an instance is identified. + * @implNote The NoArgsConstructor is private and used for deserialization + */ +@Slf4j +@EqualsAndHashCode(callSuper = false) +@NoArgsConstructor(access = AccessLevel.PROTECTED) +public abstract class PermissionOwner>> extends MetaIdentifiable implements Comparable> { + + private static final Comparator> COMPARATOR = Comparator., String>comparing( + PermissionOwner::getLabel).thenComparing(po -> po.getId().toString()); + + @Getter + @Setter + @NonNull + @NotNull + @NotEmpty + @ToString.Include + protected String name; + + @Getter + @Setter + @NonNull + @NotNull + @NotEmpty + @ToString.Include + protected String label; + + @ToString.Exclude + @NotNull + private Set permissions = new HashSet<>(); + + + @JsonCreator + public PermissionOwner(String name, String label) { + this.name = name; + this.label = label; + } + + public PermissionOwner(String name, String label, MetaStorage storage) { + this.name = name; + this.label = label; + setMetaStorage(storage); + } + + + /** + * Adds permissions to the owner object and to the persistent storage. + * + * @param permissions The permissions to add. + * @return Returns the added Permission + */ + public synchronized void addPermissions(Set permissions) { + this.permissions = ImmutableSet.builder() + .addAll(this.permissions) + .addAll( + permissions) + .build(); + updateStorage(); + } + + public synchronized void addPermission(ConqueryPermission permission) { + this.permissions = ImmutableSet.builder().addAll(this.permissions).add(permission).build(); + updateStorage(); + } + + public boolean removePermission(Permission permission) { + boolean ret = false; + synchronized (this) { + Set newSet = new HashSet<>(this.permissions); + ret = newSet.remove(permission); + this.permissions = newSet; + updateStorage(); + } + return ret; + } + + /** + * Returns a copy of the permissions hold by the owner. + * Changes to the returned collection are not persisted and do not alter the + * permissions owned. + * + * @return A set of the permissions hold by the owner. + */ + public Set getPermissions() { + if (permissions.isEmpty()) { + return Collections.emptySet(); + } + return Collections.unmodifiableSet(permissions); + + } + + /** + * Returns a collection of the effective permissions. These are the permissions of the owner and + * the permission of the roles/groups it inherits from. + * + * @return Owned and inherited permissions. + */ + @JsonIgnore + public abstract Set getEffectivePermissions(); + + public void updatePermissions(Set permissionsNew) { + synchronized (this) { + Set newSet = new HashSet<>(permissionsNew.size()); + newSet.addAll(permissionsNew); + this.permissions = newSet; + updateStorage(); + } + } + + /** + * Update this instance in the {@link MetaStorage}. + */ + protected abstract void updateStorage(); + + + @Override + public int compareTo(PermissionOwner other) { + return COMPARATOR.compare(this, other); + } + + @Override + public abstract T createId(); + + @Override + public T getId() { + return super.getId(); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Role.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Role.java index cbf7cbe0c6d..232a1749caf 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Role.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Role.java @@ -1,36 +1,36 @@ -package com.bakdata.conquery.models.auth.entities; - -import java.util.Set; - -import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; -import com.bakdata.conquery.models.identifiable.ids.specific.RoleId; -import lombok.AccessLevel; -import lombok.NoArgsConstructor; - -@NoArgsConstructor(access = AccessLevel.PRIVATE) -public class Role extends PermissionOwner { - - - public Role(String name, String label, MetaStorage storage) { - super(name, label, storage); - } - - @Override - public Set getEffectivePermissions() { - return getPermissions(); - } - - @Override - protected void updateStorage() { - getMetaStorage().updateRole(this); - - } - - @Override - public RoleId createId() { - RoleId roleId = new RoleId(name); - return roleId; - } - -} +package com.bakdata.conquery.models.auth.entities; + +import java.util.Set; + +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; +import com.bakdata.conquery.models.identifiable.ids.specific.RoleId; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class Role extends PermissionOwner { + + + public Role(String name, String label, MetaStorage storage) { + super(name, label, storage); + } + + @Override + public Set getEffectivePermissions() { + return getPermissions(); + } + + @Override + protected void updateStorage() { + getMetaStorage().updateRole(this); + + } + + @Override + public RoleId createId() { + RoleId roleId = new RoleId(name); + return roleId; + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/RoleOwner.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/RoleOwner.java index fbd38f23713..04b9b77672a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/RoleOwner.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/RoleOwner.java @@ -1,23 +1,23 @@ -package com.bakdata.conquery.models.auth.entities; - -import java.util.Set; - -import com.bakdata.conquery.models.identifiable.ids.specific.RoleId; -import com.fasterxml.jackson.annotation.JsonIgnore; - -public interface RoleOwner { - - void addRole(RoleId role); - - void removeRole(RoleId role); - - /** - * Return a copy of the roles hold by the owner. - * - * @return A set of the roles hold by the owner. - */ - //TODO migrate to NsIdRef Role - @JsonIgnore - Set getRoles(); - -} \ No newline at end of file +package com.bakdata.conquery.models.auth.entities; + +import java.util.Set; + +import com.bakdata.conquery.models.identifiable.ids.specific.RoleId; +import com.fasterxml.jackson.annotation.JsonIgnore; + +public interface RoleOwner { + + void addRole(RoleId role); + + void removeRole(RoleId role); + + /** + * Return a copy of the roles hold by the owner. + * + * @return A set of the roles hold by the owner. + */ + //TODO migrate to NsIdRef Role + @JsonIgnore + Set getRoles(); + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Subject.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Subject.java index f2e19617aa9..7d15ac85121 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Subject.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Subject.java @@ -15,7 +15,7 @@ * An interface for classes that facade a user or represent a user. * * This interface allows realms to present a user different (usually reduced) set of permissions/abilities. - * + * **/ public interface Subject extends Principal { diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/User.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/User.java index 14fbc8292e3..ef234fa8184 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/User.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/User.java @@ -1,229 +1,229 @@ -package com.bakdata.conquery.models.auth.entities; - -import java.security.Principal; -import java.util.Collection; -import java.util.Collections; -import java.util.EnumSet; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Set; -import java.util.stream.Stream; - -import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.models.auth.ConqueryAuthenticationInfo; -import com.bakdata.conquery.models.auth.permissions.Ability; -import com.bakdata.conquery.models.auth.permissions.Authorized; -import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; -import com.bakdata.conquery.models.execution.Owned; -import com.bakdata.conquery.models.identifiable.ids.specific.RoleId; -import com.bakdata.conquery.models.identifiable.ids.specific.UserId; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.Sets; -import lombok.AccessLevel; -import lombok.Getter; -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; -import org.apache.shiro.SecurityUtils; -import org.apache.shiro.authz.AuthorizationException; -import org.apache.shiro.authz.Permission; -import org.apache.shiro.subject.PrincipalCollection; - -@Slf4j -public class User extends PermissionOwner implements Principal, RoleOwner, Subject { - - @JsonProperty - private final Set roles = Collections.synchronizedSet(new HashSet<>()); - - // protected for testing purposes - @JsonIgnore - @Getter(AccessLevel.PROTECTED) - private final transient ShiroUserAdapter shiroUserAdapter; - - @JsonCreator - protected User(String name, String label) { - super(name, label); - this.shiroUserAdapter = new ShiroUserAdapter(); - } - - public User(String name, String label, MetaStorage storage) { - super(name, label, storage); - this.shiroUserAdapter = new ShiroUserAdapter(); - } - - @Override - public Set getEffectivePermissions() { - Set permissions = getPermissions(); - for (RoleId roleId : roles) { - Role role = getMetaStorage().getRole(roleId); - if (role == null) { - log.warn("Could not find role {} to gather permissions", roleId); - continue; - } - permissions = Sets.union(permissions, role.getEffectivePermissions()); - } - - try (Stream allGroups = getMetaStorage().getAllGroups()) { - - for (Iterator it = allGroups.iterator(); it.hasNext(); ) { - Group group = it.next(); - if (!group.containsUser(getId())) { - continue; - } - permissions = Sets.union(permissions, group.getEffectivePermissions()); - } - } - - - return permissions; - } - - public synchronized void addRole(RoleId role) { - if (roles.add(role)) { - log.trace("Added role {} to user {}", role, getId()); - updateStorage(); - } - } - - @Override - public void updateStorage() { - getMetaStorage().updateUser(this); - } - - @Override - public synchronized void removeRole(RoleId role) { - if (roles.remove(role)) { - log.trace("Removed role {} from user {}", role, getId()); - updateStorage(); - } - } - - public Set getRoles() { - return Collections.unmodifiableSet(roles); - } - - public void authorize(Set objects, Ability ability) { - for (Authorized object : objects) { - authorize(object, ability); - } - } - - public void authorize(@NonNull Authorized object, @NonNull Ability ability) { - if (isOwner(object)) { - return; - } - - shiroUserAdapter.checkPermission(object.createPermission(EnumSet.of(ability))); - } - - public boolean isOwner(Authorized object) { - if (object instanceof Owned owned) { - return getId().equals(owned.getOwner()); - } - - return false; - } - - @Override - public UserId createId() { - return new UserId(name); - } - - public boolean isPermittedAll(Collection authorized, Ability ability) { - return authorized.stream().allMatch(auth -> isPermitted(auth, ability)); - } - - public boolean isPermitted(Authorized object, Ability ability) { - if (isOwner(object)) { - return true; - } - - return shiroUserAdapter.isPermitted(object.createPermission(EnumSet.of(ability))); - } - - public boolean[] isPermitted(List authorizeds, Ability ability) { - boolean[] permitted = new boolean[authorizeds.size()]; - - for (int index = 0; index < authorizeds.size(); index++) { - permitted[index] = isPermitted(authorizeds.get(index), ability); - } - - return permitted; - } - - @JsonIgnore - @Override - public boolean isDisplayLogout() { - return shiroUserAdapter.getAuthenticationInfo().get().isDisplayLogout(); - } - - @JsonIgnore - @Override - public ConqueryAuthenticationInfo getAuthenticationInfo() { - return shiroUserAdapter.getAuthenticationInfo().get(); - } - - @JsonIgnore - @Override - public void setAuthenticationInfo(ConqueryAuthenticationInfo info) { - shiroUserAdapter.getAuthenticationInfo().set(info); - } - - @Override - @JsonIgnore - public User getUser() { - return this; - } - - /** - * This class is non-static, so it's a fixed part of the enclosing User object. - * It's protected for testing purposes only. - */ - @Getter - public class ShiroUserAdapter extends FilteredUser { - - private final ThreadLocal authenticationInfo = - ThreadLocal.withInitial(() -> new ConqueryAuthenticationInfo(User.this, null, null, false, null)); - - @Override - public Object getPrincipal() { - return getId(); - } - - @Override - public void checkPermission(Permission permission) throws AuthorizationException { - SecurityUtils.getSecurityManager().checkPermission(getPrincipals(), permission); - } - - @Override - public PrincipalCollection getPrincipals() { - return authenticationInfo.get().getPrincipals(); - } - - @Override - public void checkPermissions(Collection permissions) throws AuthorizationException { - SecurityUtils.getSecurityManager().checkPermissions(getPrincipals(), permissions); - } - - @Override - public boolean isPermitted(Permission permission) { - return SecurityUtils.getSecurityManager().isPermitted(getPrincipals(), permission); - } - - @Override - public boolean[] isPermitted(List permissions) { - return SecurityUtils.getSecurityManager().isPermitted(getPrincipals(), permissions); - } - - @Override - public boolean isPermittedAll(Collection permissions) { - return SecurityUtils.getSecurityManager().isPermittedAll(getPrincipals(), permissions); - } - - - } - - -} +package com.bakdata.conquery.models.auth.entities; + +import java.security.Principal; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; + +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.models.auth.ConqueryAuthenticationInfo; +import com.bakdata.conquery.models.auth.permissions.Ability; +import com.bakdata.conquery.models.auth.permissions.Authorized; +import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; +import com.bakdata.conquery.models.execution.Owned; +import com.bakdata.conquery.models.identifiable.ids.specific.RoleId; +import com.bakdata.conquery.models.identifiable.ids.specific.UserId; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.Sets; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.apache.shiro.SecurityUtils; +import org.apache.shiro.authz.AuthorizationException; +import org.apache.shiro.authz.Permission; +import org.apache.shiro.subject.PrincipalCollection; + +@Slf4j +public class User extends PermissionOwner implements Principal, RoleOwner, Subject { + + @JsonProperty + private final Set roles = Collections.synchronizedSet(new HashSet<>()); + + // protected for testing purposes + @JsonIgnore + @Getter(AccessLevel.PROTECTED) + private final transient ShiroUserAdapter shiroUserAdapter; + + @JsonCreator + protected User(String name, String label) { + super(name, label); + this.shiroUserAdapter = new ShiroUserAdapter(); + } + + public User(String name, String label, MetaStorage storage) { + super(name, label, storage); + this.shiroUserAdapter = new ShiroUserAdapter(); + } + + @Override + public Set getEffectivePermissions() { + Set permissions = getPermissions(); + for (RoleId roleId : roles) { + Role role = getMetaStorage().getRole(roleId); + if (role == null) { + log.warn("Could not find role {} to gather permissions", roleId); + continue; + } + permissions = Sets.union(permissions, role.getEffectivePermissions()); + } + + try (Stream allGroups = getMetaStorage().getAllGroups()) { + + for (Iterator it = allGroups.iterator(); it.hasNext();) { + Group group = it.next(); + if (!group.containsUser(getId())) { + continue; + } + permissions = Sets.union(permissions, group.getEffectivePermissions()); + } + } + + + return permissions; + } + + public synchronized void addRole(RoleId role) { + if (roles.add(role)) { + log.trace("Added role {} to user {}", role, getId()); + updateStorage(); + } + } + + @Override + public void updateStorage() { + getMetaStorage().updateUser(this); + } + + @Override + public synchronized void removeRole(RoleId role) { + if (roles.remove(role)) { + log.trace("Removed role {} from user {}", role, getId()); + updateStorage(); + } + } + + public Set getRoles() { + return Collections.unmodifiableSet(roles); + } + + public void authorize(Set objects, Ability ability) { + for (Authorized object : objects) { + authorize(object, ability); + } + } + + public void authorize(@NonNull Authorized object, @NonNull Ability ability) { + if (isOwner(object)) { + return; + } + + shiroUserAdapter.checkPermission(object.createPermission(EnumSet.of(ability))); + } + + public boolean isOwner(Authorized object) { + if (object instanceof Owned owned) { + return getId().equals(owned.getOwner()); + } + + return false; + } + + @Override + public UserId createId() { + return new UserId(name); + } + + public boolean isPermittedAll(Collection authorized, Ability ability) { + return authorized.stream().allMatch(auth -> isPermitted(auth, ability)); + } + + public boolean isPermitted(Authorized object, Ability ability) { + if (isOwner(object)) { + return true; + } + + return shiroUserAdapter.isPermitted(object.createPermission(EnumSet.of(ability))); + } + + public boolean[] isPermitted(List authorizeds, Ability ability) { + boolean[] permitted = new boolean[authorizeds.size()]; + + for (int index = 0; index < authorizeds.size(); index++) { + permitted[index] = isPermitted(authorizeds.get(index), ability); + } + + return permitted; + } + + @JsonIgnore + @Override + public boolean isDisplayLogout() { + return shiroUserAdapter.getAuthenticationInfo().get().isDisplayLogout(); + } + + @JsonIgnore + @Override + public ConqueryAuthenticationInfo getAuthenticationInfo() { + return shiroUserAdapter.getAuthenticationInfo().get(); + } + + @JsonIgnore + @Override + public void setAuthenticationInfo(ConqueryAuthenticationInfo info) { + shiroUserAdapter.getAuthenticationInfo().set(info); + } + + @Override + @JsonIgnore + public User getUser() { + return this; + } + + /** + * This class is non-static, so it's a fixed part of the enclosing User object. + * It's protected for testing purposes only. + */ + @Getter + public class ShiroUserAdapter extends FilteredUser { + + private final ThreadLocal authenticationInfo = ThreadLocal.withInitial( + () -> new ConqueryAuthenticationInfo(User.this, null, null, false, null)); + + @Override + public Object getPrincipal() { + return getId(); + } + + @Override + public void checkPermission(Permission permission) throws AuthorizationException { + SecurityUtils.getSecurityManager().checkPermission(getPrincipals(), permission); + } + + @Override + public PrincipalCollection getPrincipals() { + return authenticationInfo.get().getPrincipals(); + } + + @Override + public void checkPermissions(Collection permissions) throws AuthorizationException { + SecurityUtils.getSecurityManager().checkPermissions(getPrincipals(), permissions); + } + + @Override + public boolean isPermitted(Permission permission) { + return SecurityUtils.getSecurityManager().isPermitted(getPrincipals(), permission); + } + + @Override + public boolean[] isPermitted(List permissions) { + return SecurityUtils.getSecurityManager().isPermitted(getPrincipals(), permissions); + } + + @Override + public boolean isPermittedAll(Collection permissions) { + return SecurityUtils.getSecurityManager().isPermittedAll(getPrincipals(), permissions); + } + + + } + + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/IntrospectionDelegatingRealm.java b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/IntrospectionDelegatingRealm.java index 9704420b5ce..22c8c5b1450 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/IntrospectionDelegatingRealm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/IntrospectionDelegatingRealm.java @@ -74,10 +74,15 @@ public class IntrospectionDelegatingRealm extends AuthenticatingRealm implements * We only hold validated Tokens for some minutes to re-setup users and reduce fan-out. */ private LoadingCache tokenCache = CacheBuilder.newBuilder() - .expireAfterWrite(5, TimeUnit.MINUTES) - .build(new UserClaimsSetupService()); - - public IntrospectionDelegatingRealm(MetaStorage storage, IntrospectionDelegatingRealmFactory authProviderConf, KeycloakApi keycloakApi) { + .expireAfterWrite( + 5, + TimeUnit.MINUTES) + .build(new UserClaimsSetupService()); + + public IntrospectionDelegatingRealm( + MetaStorage storage, + IntrospectionDelegatingRealmFactory authProviderConf, + KeycloakApi keycloakApi) { this.storage = storage; this.authProviderConf = authProviderConf; this.keycloakApi = keycloakApi; @@ -91,8 +96,7 @@ private static String extractDisplayName(JWTClaimsSet claims) { throw new UnsupportedTokenException("Claim 'name' was empty"); } return name; - } - catch (java.text.ParseException e) { + } catch (java.text.ParseException e) { throw new IncorrectCredentialsException("Unable to extract username from token", e); } } @@ -106,7 +110,8 @@ protected void onInit() { @Override @SneakyThrows - public ConqueryAuthenticationInfo doGetAuthenticationInfo(AuthenticationToken token) throws AuthenticationException { + public ConqueryAuthenticationInfo doGetAuthenticationInfo( + AuthenticationToken token) throws AuthenticationException { if (!(TOKEN_CLASS.isAssignableFrom(token.getClass()))) { log.trace("Incompatible token. Expected {}, got {}", TOKEN_CLASS, token.getClass()); return null; @@ -123,8 +128,7 @@ public ConqueryAuthenticationInfo doGetAuthenticationInfo(AuthenticationToken to // This might fail if the tokes does not have the required audience set. try { claimsSet = tokenCache.get(bearertoken.getToken()); - } - catch (UncheckedExecutionException e) { + } catch (UncheckedExecutionException e) { final Throwable cause = e.getCause(); if (cause instanceof AuthenticationException) { throw cause; @@ -145,9 +149,10 @@ public ConqueryAuthenticationInfo doGetAuthenticationInfo(AuthenticationToken to */ private void validateToken(AuthenticationToken token) throws ParseException, IOException { // Build introspection request - TokenIntrospectionRequest - request = - new TokenIntrospectionRequest(URI.create(authProviderConf.getIntrospectionEndpoint()), authProviderConf.getClientAuthentication(), new TypelessAccessToken((String) token.getCredentials())); + TokenIntrospectionRequest request = new TokenIntrospectionRequest( + URI.create(authProviderConf.getIntrospectionEndpoint()), + authProviderConf.getClientAuthentication(), + new TypelessAccessToken((String) token.getCredentials())); // Send introspection request TokenIntrospectionResponse response = TokenIntrospectionResponse.parse(request.toHTTPRequest().send()); @@ -155,10 +160,13 @@ private void validateToken(AuthenticationToken token) throws ParseException, IOE log.trace("Retrieved token introspection response."); if (!response.indicatesSuccess()) { HTTPResponse httpResponse = response.toHTTPResponse(); - log.error("Received the following error from the auth server while validating a token: {} {} {}", httpResponse.getStatusCode(), httpResponse.getStatusMessage(), httpResponse.getContent()); + log.error( + "Received the following error from the auth server while validating a token: {} {} {}", + httpResponse.getStatusCode(), + httpResponse.getStatusMessage(), + httpResponse.getContent()); throw new AuthenticationException("Unable to retrieve access token from auth server."); - } - else if (!(response instanceof TokenIntrospectionSuccessResponse)) { + } else if (!(response instanceof TokenIntrospectionSuccessResponse)) { log.error("Unknown token response {}.", response.getClass().getName()); throw new AuthenticationException("Unknown token response. See log."); } @@ -172,7 +180,9 @@ else if (!(response instanceof TokenIntrospectionSuccessResponse)) { throw new ExpiredCredentialsException(); } - log.trace("Got an successful token introspection response: {}", log.isTraceEnabled() ? successResponse.toJSONObject().toString() : ""); + log.trace( + "Got an successful token introspection response: {}", + log.isTraceEnabled() ? successResponse.toJSONObject().toString() : ""); } @@ -193,11 +203,9 @@ private void validateAudiences(JWTClaimsSet claims) { } if (!providedAudiences.contains(expectedAudience)) { - throw new IncorrectCredentialsException("Audience does not match. Expected: '" - + expectedAudience.getValue() - + "' (was: '" - + claims.getAudience() - + "')"); + throw new IncorrectCredentialsException( + "Audience does not match. Expected: '" + expectedAudience.getValue() + "' (was: '" + claims + .getAudience() + "')"); } } @@ -239,7 +247,11 @@ private synchronized User getOrCreateUser(JWTClaimsSet claims) { log.trace("Found existing user: {}", user); // Update display name if necessary if (!user.getLabel().equals(displayName)) { - log.info("Updating display name of user [{}]: '{}' -> '{}'", user.getName(), user.getLabel(), displayName); + log.info( + "Updating display name of user [{}]: '{}' -> '{}'", + user.getName(), + user.getLabel(), + displayName); user.setLabel(displayName); user.updateStorage(); } @@ -264,10 +276,7 @@ private Set getUserGroups(JWTClaimsSet claims, String groupIdAttribute) { Set allMemberships = GroupUtil.getAllUserGroups(userGroups, groupHierarchy); // Extract eva-group-id from attributes - return allMemberships.stream() - .map(this::tryGetGroup) - .flatMap(Optional::stream) - .collect(Collectors.toSet()); + return allMemberships.stream().map(this::tryGetGroup).flatMap(Optional::stream).collect(Collectors.toSet()); } private void syncGroupMappings(UserId userId, Set mappedGroupsToDo) { @@ -280,12 +289,11 @@ private void syncGroupMappings(UserId userId, Set mappedGroupsToDo) { if (mappedGroupsToDo.contains(group)) { // Mapping is still valid, remove from todo-list mappedGroupsToDo.remove(group); - } - else { + } else { // Mapping is not valid any more remove user from group group.removeMember(userId); - } - } + } + } ); for (Group group : mappedGroupsToDo) { @@ -304,7 +312,10 @@ private Optional tryGetGroup(KeycloakGroup keycloakGroup) { // Extract group id final String groupIdString = attributes.get(authProviderConf.getGroupIdAttribute()); if (groupIdString == null) { - log.trace("Not mapping keycloak group because it has no attribute '{}': {}", authProviderConf.getGroupIdAttribute(), keycloakGroup); + log.trace( + "Not mapping keycloak group because it has no attribute '{}': {}", + authProviderConf.getGroupIdAttribute(), + keycloakGroup); return Optional.empty(); } if (Strings.isBlank(groupIdString)) { @@ -316,8 +327,7 @@ private Optional tryGetGroup(KeycloakGroup keycloakGroup) { GroupId groupId; try { groupId = GroupId.Parser.INSTANCE.parse(groupIdString); - } - catch (Exception e) { + } catch (Exception e) { log.error("Cannot parse '{}' as a GroupId. Skipping", groupIdString); return Optional.empty(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealm.java b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealm.java index 8683700d9cc..208cd452098 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealm.java @@ -1,12 +1,12 @@ package com.bakdata.conquery.models.auth.oidc; +import jakarta.validation.Validator; import java.security.PublicKey; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Supplier; -import jakarta.validation.Validator; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.ConqueryAuthenticationInfo; @@ -57,21 +57,19 @@ public class JwtPkceVerifyingRealm extends AuthenticatingRealm implements Conque * Used in handleRoleClaims as size-limited set, with LRU characteristics. * @implNote maximumSize is an arbitrary medium high number to avoid stuffing memory with token hashes, while avoiding reprocessing known access tokens. */ - private final Cache processedRoleClaims = CacheBuilder.newBuilder() - .maximumSize(1_000) - .build(); + private final Cache processedRoleClaims = CacheBuilder.newBuilder().maximumSize(1_000).build(); Supplier> idpConfigurationSupplier; public JwtPkceVerifyingRealm( - @NonNull Supplier> idpConfigurationSupplier, - @NonNull String allowedAudience, - List> additionalTokenChecks, - List alternativeIdClaims, - MetaStorage storage, - int tokenLeeway, - Validator validator + @NonNull Supplier> idpConfigurationSupplier, + @NonNull String allowedAudience, + List> additionalTokenChecks, + List alternativeIdClaims, + MetaStorage storage, + int tokenLeeway, + Validator validator ) { this.storage = storage; this.idpConfigurationSupplier = idpConfigurationSupplier; @@ -85,9 +83,11 @@ public JwtPkceVerifyingRealm( } @Override - public ConqueryAuthenticationInfo doGetAuthenticationInfo(AuthenticationToken token) throws AuthenticationException { + public ConqueryAuthenticationInfo doGetAuthenticationInfo( + AuthenticationToken token) throws AuthenticationException { - final Optional idpConfigurationOpt = idpConfigurationSupplier.get(); + final Optional idpConfigurationOpt = idpConfigurationSupplier + .get(); if (idpConfigurationOpt.isEmpty()) { log.warn("Unable to start authentication, because idp configuration is not available."); @@ -110,13 +110,16 @@ public ConqueryAuthenticationInfo doGetAuthenticationInfo(AuthenticationToken to } log.trace("Creating token verifier"); - final TokenVerifier - verifier = - TokenVerifier.create(bearerToken.getToken(), AccessToken.class) - .withChecks(new TokenVerifier.RealmUrlCheck(idpConfiguration.issuer()), TokenVerifier.SUBJECT_EXISTS_CHECK, activeVerifier) - .withChecks(tokenChecks) - .publicKey(publicKey) - .audience(allowedAudience); + final TokenVerifier verifier = TokenVerifier.create( + bearerToken.getToken(), + AccessToken.class) + .withChecks( + new TokenVerifier.RealmUrlCheck(idpConfiguration.issuer()), + TokenVerifier.SUBJECT_EXISTS_CHECK, + activeVerifier) + .withChecks(tokenChecks) + .publicKey(publicKey) + .audience(allowedAudience); log.trace("Verifying token"); @@ -124,8 +127,7 @@ public ConqueryAuthenticationInfo doGetAuthenticationInfo(AuthenticationToken to try { verifier.verify(); accessToken = verifier.getToken(); - } - catch (VerificationException e) { + } catch (VerificationException e) { log.trace("Verification failed", e); throw new IncorrectCredentialsException(e); } @@ -180,7 +182,10 @@ public ConqueryAuthenticationInfo doGetAuthenticationInfo(AuthenticationToken to * Creates a new user from values in the access token */ private User createUser(AccessToken accessToken) { - String userLabel = ObjectUtils.firstNonNull(accessToken.getName(), accessToken.getPreferredUsername(), accessToken.getSubject()); + String userLabel = ObjectUtils.firstNonNull( + accessToken.getName(), + accessToken.getPreferredUsername(), + accessToken.getSubject()); final User user = new User(accessToken.getSubject(), userLabel, storage); diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/keycloak/AccessTokenResponse.java b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/keycloak/AccessTokenResponse.java index 67003479157..6409c1ab97c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/keycloak/AccessTokenResponse.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/keycloak/AccessTokenResponse.java @@ -8,8 +8,8 @@ */ @JsonIgnoreProperties(ignoreUnknown = true) public record AccessTokenResponse( - @JsonProperty("access_token") String access_token, - @JsonProperty("expires_in") long expires_in, - @JsonProperty("token_type") String token_type + @JsonProperty("access_token") String access_token, + @JsonProperty("expires_in") long expires_in, + @JsonProperty("token_type") String token_type ) { } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/keycloak/ClientCredentials.java b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/keycloak/ClientCredentials.java index ff545a6e409..67eb7cdecc7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/keycloak/ClientCredentials.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/keycloak/ClientCredentials.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.models.auth.oidc.keycloak; import jakarta.ws.rs.core.Form; + import org.glassfish.jersey.internal.util.collection.MultivaluedStringMap; /** diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/keycloak/ClientCredentialsGrantRequestFilter.java b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/keycloak/ClientCredentialsGrantRequestFilter.java index 76a890c4ad8..1d11f9af3a2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/keycloak/ClientCredentialsGrantRequestFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/keycloak/ClientCredentialsGrantRequestFilter.java @@ -1,10 +1,5 @@ package com.bakdata.conquery.models.auth.oidc.keycloak; -import java.io.IOException; -import java.net.URI; -import java.time.Instant; -import java.time.temporal.ChronoUnit; - import jakarta.ws.rs.client.Client; import jakarta.ws.rs.client.ClientBuilder; import jakarta.ws.rs.client.ClientRequestContext; @@ -13,6 +8,11 @@ import jakarta.ws.rs.client.Invocation; import jakarta.ws.rs.core.HttpHeaders; import jakarta.ws.rs.core.MediaType; +import java.io.IOException; +import java.net.URI; +import java.time.Instant; +import java.time.temporal.ChronoUnit; + import lombok.extern.slf4j.Slf4j; /** @@ -35,8 +35,9 @@ public ClientCredentialsGrantRequestFilter(String clientId, String clientSecret, Client client = ClientBuilder.newClient(); tokenInvocation = client.target(tokenEndpoint) - .request(MediaType.APPLICATION_JSON_TYPE).buildPost(Entity.form(ClientCredentials.create(clientId - , clientSecret))); + .request(MediaType.APPLICATION_JSON_TYPE) + .buildPost( + Entity.form(ClientCredentials.create(clientId, clientSecret))); } @Override @@ -44,7 +45,10 @@ public void filter(ClientRequestContext requestContext) throws IOException { AccessTokenResponse response = getActiveAccessToken(); log.trace("Adding access token to request"); - requestContext.getHeaders().add(HttpHeaders.AUTHORIZATION, String.join(" ", response.token_type(), response.access_token())); + requestContext.getHeaders() + .add( + HttpHeaders.AUTHORIZATION, + String.join(" ", response.token_type(), response.access_token())); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/keycloak/GroupUtil.java b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/keycloak/GroupUtil.java index 3848e397dd5..abd6e341914 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/keycloak/GroupUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/keycloak/GroupUtil.java @@ -21,7 +21,9 @@ public class GroupUtil { * @return all group memberships */ @NotNull - public static Set getAllUserGroups(Set userGroups, Set groupHierarchy) { + public static Set getAllUserGroups( + Set userGroups, + Set groupHierarchy) { Set allMemberships = Collections.emptySet(); for (KeycloakGroup userGroup : userGroups) { allMemberships = Sets.union(allMemberships, GroupUtil.getParentGroups(userGroup, groupHierarchy)); @@ -68,7 +70,10 @@ public static Set getParentGroups(KeycloakGroup group, Set hierarchy, ImmutableCollection.Builder builder) { + private static void getParentGroups( + KeycloakGroup group, + Set hierarchy, + ImmutableCollection.Builder builder) { if (hierarchy == null || hierarchy.isEmpty()) { throw new NoSuchElementException("Group '" + group.path() + "' cannot be found in the hierarchy"); } @@ -84,7 +89,8 @@ private static void getParentGroups(KeycloakGroup group, Set hier if (group.path().startsWith(groupParent.path() + HIERARCHY_SEPARATOR)) { if (foundMatchingPath) { - throw new IllegalStateException("Group '" + group.path() + "' fits into multiple paths of the group hierarchy"); + throw new IllegalStateException( + "Group '" + group.path() + "' fits into multiple paths of the group hierarchy"); } foundMatchingPath = true; diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/keycloak/KeycloakApi.java b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/keycloak/KeycloakApi.java index f3870dca547..0efb9b2474f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/keycloak/KeycloakApi.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/keycloak/KeycloakApi.java @@ -1,11 +1,10 @@ package com.bakdata.conquery.models.auth.oidc.keycloak; -import java.net.URI; -import java.util.Set; - import jakarta.ws.rs.client.Client; import jakarta.ws.rs.client.WebTarget; import jakarta.ws.rs.core.GenericType; +import java.net.URI; +import java.util.Set; import com.bakdata.conquery.models.config.auth.IntrospectionDelegatingRealmFactory; import com.bakdata.conquery.util.ResourceUtil; @@ -25,7 +24,11 @@ public class KeycloakApi { public KeycloakApi(IntrospectionDelegatingRealmFactory config, Client client) { - client.register(new ClientCredentialsGrantRequestFilter(config.getClientId(), config.getClientSecret(), URI.create(config.getTokenEndpoint()))); + client.register( + new ClientCredentialsGrantRequestFilter( + config.getClientId(), + config.getClientSecret(), + URI.create(config.getTokenEndpoint()))); final WebTarget base = client.target(config.getAuthServerUrl()); @@ -40,8 +43,11 @@ public KeycloakApi(IntrospectionDelegatingRealmFactory config, Client client) { public Set getUserGroups(String userId) { Preconditions.checkNotNull(userId); - return userGroups.resolveTemplate(USER_ID_TEMPLATE, userId).request().get(new GenericType>() { - }); + return userGroups.resolveTemplate(USER_ID_TEMPLATE, userId) + .request() + .get( + new GenericType>() { + }); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/keycloak/KeycloakGroup.java b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/keycloak/KeycloakGroup.java index 9f0a2ddd61f..eb14dfbe1ab 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/keycloak/KeycloakGroup.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/keycloak/KeycloakGroup.java @@ -1,21 +1,17 @@ package com.bakdata.conquery.models.auth.oidc.keycloak; +import jakarta.validation.constraints.NotEmpty; import java.util.Map; import java.util.Set; -import jakarta.validation.constraints.NotEmpty; - import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @JsonIgnoreProperties(ignoreUnknown = true) public record KeycloakGroup( - @NotEmpty - String id, - @NotEmpty - String name, - @NotEmpty - String path, - Map attributes, - Set subGroups + @NotEmpty String id, + @NotEmpty String name, + @NotEmpty String path, + Map attributes, + Set subGroups ) { } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/passwordflow/IdpDelegatingAccessTokenCreator.java b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/passwordflow/IdpDelegatingAccessTokenCreator.java index 4ef79ab5d8a..244e0d3eb03 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/passwordflow/IdpDelegatingAccessTokenCreator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/passwordflow/IdpDelegatingAccessTokenCreator.java @@ -1,8 +1,7 @@ package com.bakdata.conquery.models.auth.oidc.passwordflow; -import java.net.URI; - import jakarta.ws.rs.core.UriBuilder; +import java.net.URI; import com.bakdata.conquery.models.auth.basic.AccessTokenCreator; import com.bakdata.conquery.models.config.auth.IntrospectionDelegatingRealmFactory; @@ -36,21 +35,29 @@ public String createAccessToken(String username, String password) { Secret passwordSecret = new Secret(password); - AuthorizationGrant grant = new ResourceOwnerPasswordCredentialsGrant(username, passwordSecret); - - URI tokenEndpoint = UriBuilder.fromUri(authProviderConf.getTokenEndpoint()).build(); + AuthorizationGrant grant = new ResourceOwnerPasswordCredentialsGrant(username, passwordSecret); + + URI tokenEndpoint = UriBuilder.fromUri(authProviderConf.getTokenEndpoint()).build(); + + TokenRequest tokenRequest = new TokenRequest( + tokenEndpoint, + authProviderConf.getClientAuthentication(), + grant, + Scope.parse("openid")); + - TokenRequest tokenRequest = new TokenRequest(tokenEndpoint, authProviderConf.getClientAuthentication(), grant, Scope.parse("openid")); - - TokenResponse response = TokenResponse.parse(tokenRequest.toHTTPRequest().send()); if (!response.indicatesSuccess()) { HTTPResponse httpResponse = response.toHTTPResponse(); - log.error("Received the following error from the auth server while validating username and password:\n\tPath: {}\n\tStatus code: {}\n\tStatus message: {}\n\tContent: {}", tokenEndpoint, httpResponse.getStatusCode(), httpResponse.getStatusMessage(), httpResponse.getContent()); + log.error( + "Received the following error from the auth server while validating username and password:\n\tPath: {}\n\tStatus code: {}\n\tStatus message: {}\n\tContent: {}", + tokenEndpoint, + httpResponse.getStatusCode(), + httpResponse.getStatusMessage(), + httpResponse.getContent()); throw new IllegalStateException("Unable to retrieve access token from auth server."); - } - else if (!(response instanceof AccessTokenResponse)) { + } else if (!(response instanceof AccessTokenResponse)) { log.error("Unknown token response {}.", response.getClass().getName()); throw new IllegalStateException("Unknown token response. See log."); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/Ability.java b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/Ability.java index 529b6ae8809..a8011b7f77f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/Ability.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/Ability.java @@ -4,23 +4,17 @@ import java.util.Set; public enum Ability { - DELETE, - READ, - CREATE, + DELETE, READ, CREATE, // Query specific - TAG, - CANCEL, - LABEL, - SHARE, + TAG, CANCEL, LABEL, SHARE, // Dataset specific STATISTIC, // Not used yet: Only get aggregations over entities and statistics about query results DOWNLOAD, // Allow download of per entity results PRESERVE_ID, // Needs extra implementation: Allow the user to see the real id of entities and externally resolve real ids into conquery - ENTITY_PREVIEW, - QUERY_PREVIEW, //TODO not yet implemented + ENTITY_PREVIEW, QUERY_PREVIEW, //TODO not yet implemented // FormConfig specific MODIFY; diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/AbilitySets.java b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/AbilitySets.java index e04a7c17f2d..0c7ec4421fa 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/AbilitySets.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/AbilitySets.java @@ -7,15 +7,28 @@ @UtilityClass public class AbilitySets { - public static final EnumSet QUERY_CREATOR = EnumSet.of(Ability.READ, Ability.DELETE, Ability.SHARE, Ability.TAG, Ability.CANCEL, Ability.LABEL); + public static final EnumSet QUERY_CREATOR = EnumSet.of( + Ability.READ, + Ability.DELETE, + Ability.SHARE, + Ability.TAG, + Ability.CANCEL, + Ability.LABEL); - public static final EnumSet - FORM_CONFIG_CREATOR = - EnumSet.of(Ability.READ, Ability.DELETE, Ability.SHARE, Ability.TAG, Ability.LABEL, Ability.MODIFY); + public static final EnumSet FORM_CONFIG_CREATOR = EnumSet.of( + Ability.READ, + Ability.DELETE, + Ability.SHARE, + Ability.TAG, + Ability.LABEL, + Ability.MODIFY); public static final EnumSet SHAREHOLDER = EnumSet.of(Ability.READ, Ability.TAG, Ability.LABEL); - public static final EnumSet - DATASET_CREATOR = - EnumSet.of(Ability.READ, Ability.DOWNLOAD, Ability.PRESERVE_ID, Ability.ENTITY_PREVIEW, Ability.QUERY_PREVIEW); + public static final EnumSet DATASET_CREATOR = EnumSet.of( + Ability.READ, + Ability.DOWNLOAD, + Ability.PRESERVE_ID, + Ability.ENTITY_PREVIEW, + Ability.QUERY_PREVIEW); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/AdminPermission.java b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/AdminPermission.java index d43ececd36a..859cacf95c6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/AdminPermission.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/AdminPermission.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.models.auth.permissions; -import java.util.Collections; import java.util.Set; import com.bakdata.conquery.io.cps.CPSType; diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/ConceptPermission.java b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/ConceptPermission.java index 1cf56d48fae..47e7b8f4877 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/ConceptPermission.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/ConceptPermission.java @@ -10,15 +10,15 @@ public class ConceptPermission extends StringPermissionBuilder { public static final String DOMAIN = "concepts"; - + public static final EnumSet ALLOWED_ABILITIES = EnumSet.of( Ability.READ ); - + public static final ConceptPermission INSTANCE = new ConceptPermission(); - - + + private ConqueryPermission instancePermission(Ability ability, ConceptId instance) { return instancePermission(ability, instance.toString()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/ConqueryPermission.java b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/ConqueryPermission.java index cb49a268e96..4c2ac9cbdec 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/ConqueryPermission.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/ConqueryPermission.java @@ -10,37 +10,37 @@ import org.apache.shiro.authz.permission.WildcardPermission; /** - * We wrap the actual interface to integrate the permissions into the + * We wrap the actual interface to integrate the permissions into the * {@link CPSType} system. * */ -@JsonTypeInfo(use=JsonTypeInfo.Id.CUSTOM, property="type") +@JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "type") @CPSBase -public interface ConqueryPermission extends Permission { - +public interface ConqueryPermission extends Permission { + /** * Gets the domains of a permission according to {@link WildcardPermission} * @return A set of the domains. */ Set getDomains(); - + /** * Gets the abilities of a permission according to {@link WildcardPermission} * @return A set of the abilities. */ Set getAbilities(); - + /** * Gets the instances of a permission according to {@link WildcardPermission} * @return A set of the instances. */ Set getInstances(); - + /** * Returns the creation time of the permission. * @return The creation time as an {@link Instant}. */ Instant getCreationTime(); - - + + } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/DatasetPermission.java b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/DatasetPermission.java index d3e9d3719b8..3acde9564e8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/DatasetPermission.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/DatasetPermission.java @@ -12,13 +12,13 @@ public class DatasetPermission extends StringPermissionBuilder { public static final String DOMAIN = "datasets"; private static final Set ALLOWED_ABILITIES = AbilitySets.DATASET_CREATOR; - + public static final DatasetPermission INSTANCE = new DatasetPermission(); - + public ConqueryPermission instancePermission(Ability ability, DatasetId instance) { return instancePermission(ability, instance.toString()); } - + public ConqueryPermission instancePermission(Set abilities, DatasetId instance) { return instancePermission(abilities, instance.toString()); } @@ -32,8 +32,8 @@ public String getDomain() { public Set getAllowedAbilities() { return ALLOWED_ABILITIES; } - - + + //// Helper functions public static ConqueryPermission onInstance(Set abilities, DatasetId instance) { return INSTANCE.instancePermission(abilities, instance); diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/ExecutionPermission.java b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/ExecutionPermission.java index c9b84663635..4c1b93454d5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/ExecutionPermission.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/ExecutionPermission.java @@ -13,12 +13,12 @@ public class ExecutionPermission extends StringPermissionBuilder { public static final String DOMAIN = "queries"; public static final EnumSet ALLOWED_ABILITIES = EnumSet.of( - Ability.READ, - Ability.DELETE, - Ability.SHARE, - Ability.TAG, - Ability.CANCEL, - Ability.LABEL + Ability.READ, + Ability.DELETE, + Ability.SHARE, + Ability.TAG, + Ability.CANCEL, + Ability.LABEL ); public static final ExecutionPermission INSTANCE = new ExecutionPermission(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/FormConfigPermission.java b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/FormConfigPermission.java index 519c0c0a2cc..72e8ba56fee 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/FormConfigPermission.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/FormConfigPermission.java @@ -11,13 +11,13 @@ public class FormConfigPermission extends StringPermissionBuilder { public static final String DOMAIN = "form-config"; - + public static final EnumSet ALLOWED_ABILITIES = AbilitySets.FORM_CONFIG_CREATOR; - + public static final FormConfigPermission INSTANCE = new FormConfigPermission(); - - + + private ConqueryPermission instancePermission(Ability ability, FormConfigId instance) { return instancePermission(ability, instance.toString()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/FormPermission.java b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/FormPermission.java index 91daa31c9c3..da694350207 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/FormPermission.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/FormPermission.java @@ -12,12 +12,12 @@ */ @CPSType(id = "FORM_TYPE", base = StringPermissionBuilder.class) public class FormPermission extends StringPermissionBuilder { - + public static final String DOMAIN = "forms"; private static final Set ALLOWED_ABILITIES = Set.of( Ability.CREATE); - + public final static FormPermission INSTANCE = new FormPermission(); @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/StringPermissionBuilder.java b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/StringPermissionBuilder.java index 09b53ba6867..257f066f3ac 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/StringPermissionBuilder.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/StringPermissionBuilder.java @@ -15,61 +15,72 @@ */ @CPSBase public abstract class StringPermissionBuilder { - + private static final String PART_DIVIDER_TOKEN = ":"; private static final String SUBPART_DIVIDER_TOKEN = ","; - private final static String DOMAIN_FORMAT = "%s"; - private final static String ABILITY_FORMAT = "%s"+PART_DIVIDER_TOKEN+"%s"; - private final static String INSTANCE_FORMAT = "%s"+PART_DIVIDER_TOKEN+"%s"+PART_DIVIDER_TOKEN+"%s"; + private final static String DOMAIN_FORMAT = "%s"; + private final static String ABILITY_FORMAT = "%s" + PART_DIVIDER_TOKEN + "%s"; + private final static String INSTANCE_FORMAT = "%s" + PART_DIVIDER_TOKEN + "%s" + PART_DIVIDER_TOKEN + "%s"; - //// DOMAIN //// + //// DOMAIN //// protected WildcardPermission domainPermission() { return new WildcardPermission(String.format(DOMAIN_FORMAT, getDomain())); } - - //// ABILITY on DOMAIN //// - protected WildcardPermission abilityPermission(Ability ability) { - ensureAllowedAbility(getAllowedAbilities(),ability); + + //// ABILITY on DOMAIN + protected WildcardPermission abilityPermission(Ability ability) { + ensureAllowedAbility(getAllowedAbilities(), ability); return new WildcardPermission(String.format(ABILITY_FORMAT, getDomain(), ability)); } protected WildcardPermission abilityPermission(Set abilities) { - ensureAllowedAbilities(getAllowedAbilities(),abilities); - return new WildcardPermission(String.format(ABILITY_FORMAT, getDomain(), abilities.stream().map(Ability::toString).collect(Collectors.joining(SUBPART_DIVIDER_TOKEN)))); + ensureAllowedAbilities(getAllowedAbilities(), abilities); + return new WildcardPermission( + String.format( + ABILITY_FORMAT, + getDomain(), + abilities.stream().map(Ability::toString).collect(Collectors.joining(SUBPART_DIVIDER_TOKEN)))); } - //// ABILITY on INSTANCE in DOMAIN //// + //// ABILITY on INSTANCE in DOMAIN protected WildcardPermission instancePermission(Ability ability, String instance) { - ensureAllowedAbility(getAllowedAbilities(),ability); + ensureAllowedAbility(getAllowedAbilities(), ability); return new WildcardPermission(String.format(INSTANCE_FORMAT, getDomain(), ability, instance)); } protected WildcardPermission instancePermission(Set abilities, String instance) { - ensureAllowedAbilities(getAllowedAbilities(),abilities); - return new WildcardPermission(String.format(INSTANCE_FORMAT, getDomain(), abilities.stream().map(Ability::toString).collect(Collectors.joining(SUBPART_DIVIDER_TOKEN)) , instance)); + ensureAllowedAbilities(getAllowedAbilities(), abilities); + return new WildcardPermission( + String.format( + INSTANCE_FORMAT, + getDomain(), + abilities.stream().map(Ability::toString).collect(Collectors.joining(SUBPART_DIVIDER_TOKEN)), + instance)); } //// UTIL //// private static void ensureAllowedAbility(Set allowedAbilities, Ability ability) { - if(!allowedAbilities.contains(ability)) { - throw new IllegalArgumentException(String.format("Ability %s is not allowed. Allowed abilities:", ability, allowedAbilities)); + if (!allowedAbilities.contains(ability)) { + throw new IllegalArgumentException( + String.format("Ability %s is not allowed. Allowed abilities:", ability, allowedAbilities)); } } - - private static void ensureAllowedAbilities(Set allowedAbilities, Set abilities) { - if(!allowedAbilities.containsAll(abilities)) { - throw new IllegalArgumentException(String.format("Abilities %s are not allowed. Allowed abilities:", abilities, allowedAbilities)); + + private static void ensureAllowedAbilities(Set allowedAbilities, Set abilities) { + if (!allowedAbilities.containsAll(abilities)) { + throw new IllegalArgumentException( + String.format("Abilities %s are not allowed. Allowed abilities:", abilities, allowedAbilities)); } } - + /** * The domain a permission grants access on (e.g. datasets). * The domain name must not include PART_DIVIDER_TOKEN or the SUBPART_DIVIDER_TOKEN. * @return The domain name. */ public abstract String getDomain(); - - + + /** * The Abilities, that are allowed for the domain. * @return A set of the allowed abilities. diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/util/SkippingCredentialsMatcher.java b/backend/src/main/java/com/bakdata/conquery/models/auth/util/SkippingCredentialsMatcher.java index 770a6c4fae9..b02b57f67ae 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/util/SkippingCredentialsMatcher.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/util/SkippingCredentialsMatcher.java @@ -16,5 +16,5 @@ public enum SkippingCredentialsMatcher implements CredentialsMatcher { public boolean doCredentialsMatch(AuthenticationToken token, AuthenticationInfo info) { return true; } - -} \ No newline at end of file + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/util/SubjectPrincipalCollection.java b/backend/src/main/java/com/bakdata/conquery/models/auth/util/SubjectPrincipalCollection.java index 1b081b9e834..ce5d70dcc94 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/util/SubjectPrincipalCollection.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/util/SubjectPrincipalCollection.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.auth.util; +import java.util.*; + import com.bakdata.conquery.models.auth.ConqueryAuthenticationRealm; import com.bakdata.conquery.models.auth.entities.Subject; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -7,8 +9,6 @@ import lombok.NonNull; import org.apache.shiro.subject.PrincipalCollection; -import java.util.*; - /** * Principal collection that carries a ready to use {@link Subject} object. * @@ -27,19 +27,21 @@ public SubjectPrincipalCollection(@NonNull Subject subject, ConqueryAuthenticati this.realm = realm; } - @Override @JsonIgnore + @Override + @JsonIgnore public Iterator iterator() { return Collections.singleton(principal).iterator(); } - @Override @JsonIgnore + @Override + @JsonIgnore public Subject getPrimaryPrincipal() { return principal; } @Override public T oneByType(Class type) { - if(type.isAssignableFrom(principal.getClass())) { + if (type.isAssignableFrom(principal.getClass())) { return (T) principal; } return null; @@ -47,15 +49,15 @@ public T oneByType(Class type) { @Override public Collection byType(Class type) { - if(type.isAssignableFrom(principal.getClass())) { - return List.of((T)principal); + if (type.isAssignableFrom(principal.getClass())) { + return List.of((T) principal); } return Collections.emptyList(); } @Override public List asList() { - return List.of(principal); + return List.of(principal); } @Override @@ -65,21 +67,23 @@ public Set asSet() { @Override public Collection fromRealm(String realmName) { - if(realm.getName().equals(realmName)){ + if (realm.getName().equals(realmName)) { return List.of(principal); } return Collections.emptyList(); } - @Override @JsonIgnore + @Override + @JsonIgnore public Set getRealmNames() { - if(realm != null) { + if (realm != null) { return Set.of(realm.getName()); } return Collections.emptySet(); } - @Override @JsonIgnore + @Override + @JsonIgnore public boolean isEmpty() { return false; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/web/AuthCookieFilter.java b/backend/src/main/java/com/bakdata/conquery/models/auth/web/AuthCookieFilter.java index 25ab71e036b..f4d1b7a5291 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/web/AuthCookieFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/web/AuthCookieFilter.java @@ -2,10 +2,6 @@ import static com.bakdata.conquery.models.auth.web.AuthCookieFilter.PRIORITY; -import java.io.IOException; - -import com.bakdata.conquery.models.config.ConqueryConfig; -import com.google.common.base.Strings; import jakarta.annotation.Priority; import jakarta.inject.Inject; import jakarta.ws.rs.Priorities; @@ -18,6 +14,10 @@ import jakarta.ws.rs.core.HttpHeaders; import jakarta.ws.rs.core.NewCookie; import jakarta.ws.rs.core.UriBuilder; +import java.io.IOException; + +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.google.common.base.Strings; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.eclipse.jetty.http.HttpHeader; @@ -58,13 +58,13 @@ public void filter(ContainerRequestContext requestContext) throws IOException { String queryToken = requestContext.getUriInfo().getQueryParameters().getFirst(ACCESS_TOKEN); - if(!cookie.getValue().isEmpty() && queryToken != null) { + if (!cookie.getValue().isEmpty() && queryToken != null) { log.trace("Ignoring cookie"); return; } - + // Get the token from the cookie and put it into the header - requestContext.getHeaders().add(HttpHeaders.AUTHORIZATION,PREFIX + " " + cookie.getValue()); + requestContext.getHeaders().add(HttpHeaders.AUTHORIZATION, PREFIX + " " + cookie.getValue()); // Remove the cookie for the rest of this processing requestContext.getCookies().remove(ACCESS_TOKEN); // Remove the query parameter @@ -87,10 +87,11 @@ public void filter(ContainerRequestContext request, ContainerResponseContext res log.debug("Overwriting {} cookie", ACCESS_TOKEN); } final NewCookie authCookie = config.getAuthentication().createAuthCookie(request, token); - response.getHeaders().add( + response.getHeaders() + .add( HttpHeader.SET_COOKIE.toString(), authCookie - ); + ); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/web/AuthFilter.java b/backend/src/main/java/com/bakdata/conquery/models/auth/web/AuthFilter.java index a2c278646a7..af5c20e289f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/web/AuthFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/web/AuthFilter.java @@ -1,8 +1,5 @@ package com.bakdata.conquery.models.auth.web; -import java.io.IOException; -import java.util.HashSet; -import java.util.Set; import jakarta.annotation.Priority; import jakarta.inject.Inject; import jakarta.ws.rs.NotAuthorizedException; @@ -10,6 +7,9 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.container.PreMatching; import jakarta.ws.rs.core.SecurityContext; +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; import com.bakdata.conquery.models.auth.ConqueryAuthenticationRealm; import com.bakdata.conquery.models.auth.ConqueryAuthenticator; @@ -78,8 +78,7 @@ public void filter(final ContainerRequestContext requestContext) throws IOExcept // Success an extracted token could be authenticated log.trace("Authentication was successful for token type {}", token.getClass().getName()); return; - } - catch (AuthenticationException e) { + } catch (AuthenticationException e) { // This is the shiro way to indicate that authentication failed failedTokens++; log.trace("Token authentication failed:", e); diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/web/AuthenticationExceptionMapper.java b/backend/src/main/java/com/bakdata/conquery/models/auth/web/AuthenticationExceptionMapper.java index f1ab979cb38..aa2173e2ea9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/web/AuthenticationExceptionMapper.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/web/AuthenticationExceptionMapper.java @@ -3,6 +3,7 @@ import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.ext.ExceptionMapper; + import lombok.extern.slf4j.Slf4j; import org.apache.shiro.authc.AuthenticationException; @@ -17,8 +18,9 @@ public Response toResponse(AuthenticationException exception) { log.warn("Shiro failed to authenticate the request. See the following trace:", exception); return Response.status(Response.Status.UNAUTHORIZED) .type(MediaType.APPLICATION_JSON_TYPE) - .entity("An authentication error occurred. The error has been logged") + .entity( + "An authentication error occurred. The error has been logged") .build(); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/web/AuthorizationExceptionMapper.java b/backend/src/main/java/com/bakdata/conquery/models/auth/web/AuthorizationExceptionMapper.java index 4162a49ca2f..e476ded9ac3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/web/AuthorizationExceptionMapper.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/web/AuthorizationExceptionMapper.java @@ -3,12 +3,13 @@ import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.ext.ExceptionMapper; + import lombok.extern.slf4j.Slf4j; import org.apache.shiro.authz.AuthorizationException; /** * This mapper intercepts all {@link AuthorizationException}s that occur during a request. - * It then logs the specific cause and returns a {@link Response.Status.UNAUTHORIZED} to the client. + * It then logs the specific cause and returns a {@link Response.Status#FORBIDDEN} to the client. * */ @Slf4j @@ -21,7 +22,8 @@ public Response toResponse(AuthorizationException exception) { log.trace("Shiro failed to authorize the request.", exception); return Response.status(Response.Status.FORBIDDEN) .type(MediaType.APPLICATION_JSON_TYPE) - .entity("Not sufficient permissions to perform action: " + exception.getMessage()) + .entity( + "Not sufficient permissions to perform action: " + exception.getMessage()) .build(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/web/RedirectingAuthFilter.java b/backend/src/main/java/com/bakdata/conquery/models/auth/web/RedirectingAuthFilter.java index 972569a88b5..19e7224beb6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/web/RedirectingAuthFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/web/RedirectingAuthFilter.java @@ -1,10 +1,5 @@ package com.bakdata.conquery.models.auth.web; -import java.io.IOException; -import java.net.URI; -import java.util.ArrayList; -import java.util.List; -import java.util.function.Function; import jakarta.annotation.Priority; import jakarta.inject.Inject; import jakarta.ws.rs.BadRequestException; @@ -16,6 +11,11 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.container.PreMatching; import jakarta.ws.rs.core.Response; +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.resources.admin.ui.model.UIView; @@ -64,25 +64,27 @@ public class RedirectingAuthFilter extends io.dropwizard.auth.AuthFilter 1) { + } else if (authenticatedRedirects.size() > 1) { log.error("Multiple authenticated redirects generated. Only one should be possible"); - throw new BadRequestException("The request triggered more than one multistep authentication schema, which is not allowed."); + throw new BadRequestException( + "The request triggered more than one multistep authentication schema, which is not allowed."); } // The request was not authenticated, nor was it a step towards an authentication, so we redirect the user to a login. @@ -143,7 +145,8 @@ else if (authenticatedRedirects.size() > 1) { } // Give the user a choice to choose between them. - throw new WebApplicationException(Response.ok(new UIView<>("logins.html.ftl", null, loginRedirects)).build()); + throw new WebApplicationException( + Response.ok(new UIView<>("logins.html.ftl", null, loginRedirects)).build()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/web/csrf/CsrfTokenCheckFilter.java b/backend/src/main/java/com/bakdata/conquery/models/auth/web/csrf/CsrfTokenCheckFilter.java index 910af59848d..eaa59434be3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/web/csrf/CsrfTokenCheckFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/web/csrf/CsrfTokenCheckFilter.java @@ -1,15 +1,15 @@ package com.bakdata.conquery.models.auth.web.csrf; -import java.io.IOException; -import java.util.Optional; - -import com.bakdata.conquery.models.auth.web.AuthCookieFilter; import jakarta.annotation.Priority; import jakarta.ws.rs.ForbiddenException; import jakarta.ws.rs.HttpMethod; import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.container.ContainerRequestFilter; import jakarta.ws.rs.core.Cookie; +import java.io.IOException; +import java.util.Optional; + +import com.bakdata.conquery.models.auth.web.AuthCookieFilter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -25,9 +25,8 @@ public class CsrfTokenCheckFilter implements ContainerRequestFilter { @Override public void filter(ContainerRequestContext requestContext) throws IOException { - final String - cookieTokenHash = - Optional.ofNullable(requestContext.getCookies().get(CsrfTokenSetFilter.CSRF_COOKIE_NAME)).map(Cookie::getValue).orElse(null); + final String cookieTokenHash = Optional.ofNullable( + requestContext.getCookies().get(CsrfTokenSetFilter.CSRF_COOKIE_NAME)).map(Cookie::getValue).orElse(null); final String headerToken = requestContext.getHeaders().getFirst(CSRF_TOKEN_HEADER); final String method = requestContext.getMethod(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/web/csrf/CsrfTokenSetFilter.java b/backend/src/main/java/com/bakdata/conquery/models/auth/web/csrf/CsrfTokenSetFilter.java index 939db8a213b..1aeffa606d0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/web/csrf/CsrfTokenSetFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/web/csrf/CsrfTokenSetFilter.java @@ -1,5 +1,12 @@ package com.bakdata.conquery.models.auth.web.csrf; +import jakarta.annotation.Priority; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.container.ContainerRequestFilter; +import jakarta.ws.rs.container.ContainerResponseContext; +import jakarta.ws.rs.container.ContainerResponseFilter; +import jakarta.ws.rs.core.HttpHeaders; +import jakarta.ws.rs.core.NewCookie; import java.io.IOException; import java.security.SecureRandom; import java.util.Base64; @@ -11,13 +18,6 @@ import com.password4j.PBKDF2Function; import com.password4j.Password; import com.password4j.types.Hmac; -import jakarta.annotation.Priority; -import jakarta.ws.rs.container.ContainerRequestContext; -import jakarta.ws.rs.container.ContainerRequestFilter; -import jakarta.ws.rs.container.ContainerResponseContext; -import jakarta.ws.rs.container.ContainerResponseFilter; -import jakarta.ws.rs.core.HttpHeaders; -import jakarta.ws.rs.core.NewCookie; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; @@ -53,14 +53,22 @@ public class CsrfTokenSetFilter implements ContainerRequestFilter, ContainerResp @Override public void filter(ContainerRequestContext requestContext) throws IOException { - final String token = RandomStringUtils.random(TOKEN_LENGTH, 0, 0, true, true, - null, random + final String token = RandomStringUtils.random( + TOKEN_LENGTH, + 0, + 0, + true, + true, + null, + random ); requestContext.setProperty(CSRF_TOKEN_PROPERTY, token); } @Override - public void filter(ContainerRequestContext requestContext, ContainerResponseContext responseContext) throws IOException { + public void filter( + ContainerRequestContext requestContext, + ContainerResponseContext responseContext) throws IOException { final String csrfToken = getCsrfTokenProperty(requestContext); if (StringUtils.isBlank(csrfToken)) { @@ -73,27 +81,29 @@ public void filter(ContainerRequestContext requestContext, ContainerResponseCont log.trace("Hashed token for cookie. token='{}' hash='{}'", csrfToken, csrfTokenHash); responseContext.getHeaders() - .add(HttpHeaders.SET_COOKIE, new NewCookie( - CSRF_COOKIE_NAME, - csrfTokenHash, - "/", - null, - 0, - null, - COOKIE_MAX_AGE, - null, - requestContext.getSecurityContext().isSecure(), - false - )); + .add( + HttpHeaders.SET_COOKIE, + new NewCookie( + CSRF_COOKIE_NAME, + csrfTokenHash, + "/", + null, + 0, + null, + COOKIE_MAX_AGE, + null, + requestContext.getSecurityContext().isSecure(), + false + )); } private static String getTokenHash(String csrfToken) { final Stopwatch stopwatch = log.isTraceEnabled() ? Stopwatch.createStarted() : null; final Hash hash = Password.hash(csrfToken).addRandomSalt(SALT_LENGTH).with(HASH_FUNCTION); - + log.trace("Generated token in {}", stopwatch); - + final String encodedSalt = Base64.getEncoder().encodeToString(hash.getSaltBytes()); // Use '_' as join char, because it is not part of the standard base64 encoding (in base64url though) @@ -103,7 +113,8 @@ private static String getTokenHash(String csrfToken) { public static boolean checkHash(String token, String hash) { int delimIdx; if ((delimIdx = hash.indexOf("_")) == -1 || delimIdx == hash.length() - 1) { - throw new IllegalArgumentException("The provided hash must be of this form: _, was: " + hash); + throw new IllegalArgumentException( + "The provided hash must be of this form: _, was: " + hash); } final String encodedSalt = hash.substring(0, delimIdx); final String saltedHash = hash.substring(delimIdx + 1); diff --git a/backend/src/main/java/com/bakdata/conquery/models/common/CDate.java b/backend/src/main/java/com/bakdata/conquery/models/common/CDate.java index 5e496d5ec5b..a6040209fa6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/common/CDate.java +++ b/backend/src/main/java/com/bakdata/conquery/models/common/CDate.java @@ -18,9 +18,9 @@ public final class CDate { public int ofLocalDate(LocalDate date) { return Ints.checkedCast(date.toEpochDay()); } - + public int ofLocalDate(LocalDate date, int defaultValue) { - if(date == null) { + if (date == null) { return defaultValue; } return Ints.checkedCast(date.toEpochDay()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/common/CDateSet.java b/backend/src/main/java/com/bakdata/conquery/models/common/CDateSet.java index 25c57a47941..65206fc8710 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/common/CDateSet.java +++ b/backend/src/main/java/com/bakdata/conquery/models/common/CDateSet.java @@ -34,7 +34,7 @@ public static CDateSet createEmpty() { return new CDateSet(new TreeMap<>()); } - + public static CDateSet createFull() { CDateSet set = new CDateSet(new TreeMap<>()); set.add(CDateRange.all()); @@ -46,7 +46,7 @@ public static CDateSet create(CDateSet rangeSet) { result.addAll(rangeSet); return result; } - + public static CDateSet create(CDateRange range) { CDateSet result = createEmpty(); result.add(range); @@ -102,8 +102,7 @@ public boolean equals(Object object) { try { return this.size() == other.size() && this.containsAll(other); - } - catch (NullPointerException | ClassCastException ignored) { + } catch (NullPointerException | ClassCastException ignored) { return false; } } @@ -116,8 +115,7 @@ public CDateRange rangeContaining(int value) { Entry floorEntry = rangesByLowerBound.floorEntry(value); if (floorEntry != null && floorEntry.getValue().contains(value)) { return floorEntry.getValue(); - } - else { + } else { return null; } } @@ -139,7 +137,7 @@ public boolean contains(LocalDate value) { public boolean contains(int value) { return rangeContaining(value) != null; } - + public boolean isEmpty() { return asRanges().isEmpty(); } @@ -148,11 +146,11 @@ public void clear() { rangesByLowerBound.clear(); } - + public void addAll(CDateSet other) { addAll(other.asRanges()); } - + public void removeAll(CDateSet other) { removeAll(other.asRanges()); } @@ -177,7 +175,7 @@ public void removeAll(Iterable ranges) { remove(range); } } - + public boolean intersects(CDateRange range) { checkNotNull(range); Entry ceilingEntry = rangesByLowerBound.ceilingEntry(range.getMinValue()); @@ -199,12 +197,14 @@ public CDateRange span() { if (firstEntry == null) { throw new NoSuchElementException(); } - return CDateRange.of(firstEntry.getValue().getMinValue(), rangesByLowerBound.lastEntry().getValue().getMaxValue()); + return CDateRange.of( + firstEntry.getValue().getMinValue(), + rangesByLowerBound.lastEntry().getValue().getMaxValue()); } public void add(CDateRange rangeToAdd) { checkNotNull(rangeToAdd); - + int lbToAdd = rangeToAdd.getMinValue(); int ubToAdd = rangeToAdd.getMaxValue(); @@ -218,7 +218,7 @@ public void add(CDateRange rangeToAdd) { if (rangeBelowLB.getMaxValue() > ubToAdd) { ubToAdd = rangeBelowLB.getMaxValue(); } - + } } @@ -235,7 +235,7 @@ public void add(CDateRange rangeToAdd) { putRange(CDateRange.of(lbToAdd, ubToAdd)); } - + public void remove(CDateRange rangeToRemove) { checkNotNull(rangeToRemove); @@ -245,7 +245,7 @@ public void remove(CDateRange rangeToRemove) { //left neighbor intersects removed range => shorten it to everything before removed range if (rangeBelowLB.getMaxValue() >= rangeToRemove.getMinValue()) { putRange(CDateRange.of(rangeBelowLB.getMinValue(), rangeToRemove.getMinValue() - 1)); - + //left neighbor reaches beyond removed range => have to add cut of right part if (rangeBelowLB.getMaxValue() > rangeToRemove.getMaxValue()) { putRange(CDateRange.of(rangeToRemove.getMaxValue() + 1, rangeBelowLB.getMaxValue())); @@ -263,7 +263,9 @@ public void remove(CDateRange rangeToRemove) { } } - rangesByLowerBound.subMap(rangeToRemove.getMinValue(), IntMath.saturatedAdd(rangeToRemove.getMaxValue(),1)).clear(); + rangesByLowerBound.subMap( + rangeToRemove.getMinValue(), + IntMath.saturatedAdd(rangeToRemove.getMaxValue(), 1)).clear(); } private void putRange(CDateRange range) { @@ -275,22 +277,22 @@ public void maskedAdd(CDateRange toAdd, CDateSet mask) { maskedAdd(toAdd, mask, CDateRange.POSITIVE_INFINITY); } - public void maskedAdd(CDateRange toAdd, CDateSet mask, int truncateMax){ - if(mask.isEmpty()){ + public void maskedAdd(CDateRange toAdd, CDateSet mask, int truncateMax) { + if (mask.isEmpty()) { return; } - if(mask.isAll()){ + if (mask.isAll()) { add(toAdd); return; } - if(toAdd.isAll()){ + if (toAdd.isAll()) { addAll(mask); return; } - if(toAdd.isExactly() && mask.contains(toAdd.getMinValue())){ + if (toAdd.isExactly() && mask.contains(toAdd.getMinValue())) { add(toAdd); return; } @@ -302,21 +304,21 @@ public void maskedAdd(CDateRange toAdd, CDateSet mask, int truncateMax){ search = mask.rangesByLowerBound.floorKey(toAdd.getMinValue()); } - if(search == null) { + if (search == null) { search = mask.rangesByLowerBound.firstKey(); } Integer searchEnd = null; - if(toAdd.hasUpperBound()){ + if (toAdd.hasUpperBound()) { searchEnd = mask.rangesByLowerBound.floorKey(toAdd.getMaxValue()); } - if(searchEnd == null){ + if (searchEnd == null) { searchEnd = mask.rangesByLowerBound.lastKey(); } - while(search != null && search <= searchEnd) { + while (search != null && search <= searchEnd) { final CDateRange range = mask.rangesByLowerBound.get(search); search = mask.rangesByLowerBound.higherKey(search); @@ -324,22 +326,22 @@ public void maskedAdd(CDateRange toAdd, CDateSet mask, int truncateMax){ int lowerBound = range.getMinValue(); int upperBound = range.getMaxValue(); - if(upperBound < toAdd.getMinValue()){ + if (upperBound < toAdd.getMinValue()) { continue; } - if(lowerBound < toAdd.getMinValue()){ + if (lowerBound < toAdd.getMinValue()) { lowerBound = toAdd.getMinValue(); } - if(upperBound > toAdd.getMaxValue()){ + if (upperBound > toAdd.getMaxValue()) { upperBound = toAdd.getMaxValue(); } upperBound = Math.min(upperBound, truncateMax); // value was not contained - if(lowerBound > upperBound){ + if (lowerBound > upperBound) { continue; } @@ -358,7 +360,7 @@ public String toString() { @JsonIgnore public boolean isAll() { - if(this.rangesByLowerBound.isEmpty()) { + if (this.rangesByLowerBound.isEmpty()) { return false; } return this.rangesByLowerBound.firstEntry().getValue().isAll(); @@ -369,30 +371,32 @@ public boolean isAll() { */ @JsonIgnore public boolean isOpen() { - if(this.rangesByLowerBound.isEmpty()) { + if (this.rangesByLowerBound.isEmpty()) { return false; } // Since we might be all, just check if any of the boundaries are open. - return rangesByLowerBound.firstEntry().getValue().isOpen() || rangesByLowerBound.lastEntry().getValue().isOpen(); + return rangesByLowerBound.firstEntry().getValue().isOpen() || rangesByLowerBound.lastEntry() + .getValue() + .isOpen(); } public void retainAll(CDateSet retained) { - if(retained.isEmpty()) { + if (retained.isEmpty()) { this.clear(); return; } - if(retained.isAll()) { + if (retained.isAll()) { return; } List l = new ArrayList<>(retained.rangesByLowerBound.values()); - + //remove all before the first range - if(!l.get(0).isAtMost()) { + if (!l.get(0).isAtMost()) { this.remove(CDateRange.atMost(l.get(0).getMinValue() - 1)); } - + //remove all between ranges for (int i = 0; i < l.size() - 1; i++) { this.remove(CDateRange.of(l.get(i).getMaxValue() + 1, l.get(i + 1).getMinValue() - 1)); @@ -403,19 +407,19 @@ public void retainAll(CDateSet retained) { this.remove(CDateRange.atLeast(l.get(l.size() - 1).getMaxValue() + 1)); } } - + public void retainAll(CDateRange retained) { - if(retained.isAll()) { + if (retained.isAll()) { return; } //remove all before the range - if(!retained.isAtMost()) { + if (!retained.isAtMost()) { this.remove(CDateRange.atMost(retained.getMinValue() - 1)); } - + //remove all after the Range - if(!retained.isAtLeast()) { + if (!retained.isAtLeast()) { this.remove(CDateRange.atLeast(retained.getMaxValue() + 1)); } } @@ -426,15 +430,15 @@ public void retainAll(CDateRange retained) { */ public Long countDays() { //if we have no entries we return zero days - if(rangesByLowerBound.firstEntry() == null) { + if (rangesByLowerBound.firstEntry() == null) { return 0L; } - if(rangesByLowerBound.firstEntry().getValue().isOpen() || rangesByLowerBound.lastEntry().getValue().isOpen()) { + if (rangesByLowerBound.firstEntry().getValue().isOpen() || rangesByLowerBound.lastEntry().getValue().isOpen()) { return null; } long sum = 0; - for(CDateRange r:this.asRanges()) { - sum+=r.getMaxValue() - r.getMinValue() + 1; + for (CDateRange r : this.asRanges()) { + sum += r.getMaxValue() - r.getMinValue() + 1; } return sum; } @@ -447,4 +451,4 @@ public int getMaxValue() { return rangesByLowerBound.lastEntry().getValue().getMaxValue(); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/common/CQuarter.java b/backend/src/main/java/com/bakdata/conquery/models/common/CQuarter.java index acfb5529cf1..bf3211119fe 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/common/CQuarter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/common/CQuarter.java @@ -10,8 +10,8 @@ public final class CQuarter { public CDateRange toRange(int value) { return CDateRange.of(value, getLastDay(value)); } - + public int getLastDay(int firstEpoch) { return Ints.checkedCast(QuarterUtils.getLastDayOfQuarter(firstEpoch).toEpochDay()); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/common/ColumnUtils.java b/backend/src/main/java/com/bakdata/conquery/models/common/ColumnUtils.java index 5cc71b2b741..23b37ece71c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/common/ColumnUtils.java +++ b/backend/src/main/java/com/bakdata/conquery/models/common/ColumnUtils.java @@ -1,12 +1,12 @@ package com.bakdata.conquery.models.common; +import java.util.Set; + import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; import lombok.extern.slf4j.Slf4j; -import java.util.Set; - @Slf4j public class ColumnUtils { public static boolean assertValidColumnTypes(ColumnId column, Set acceptedColumnTypes) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/common/IRange.java b/backend/src/main/java/com/bakdata/conquery/models/common/IRange.java index 073da173890..d12ab7308d0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/common/IRange.java +++ b/backend/src/main/java/com/bakdata/conquery/models/common/IRange.java @@ -15,27 +15,25 @@ public interface IRange toGuavaRange() { - if(isAtLeast()) { + if (isAtLeast()) { return com.google.common.collect.Range.atLeast(getMin()); } - if(isAtMost()) { + if (isAtMost()) { return com.google.common.collect.Range.atMost(getMax()); } - if(isAll()) { + if (isAll()) { return com.google.common.collect.Range.all(); } return com.google.common.collect.Range.closed(getMin(), getMax()); } default boolean intersects(CLASS other) { - if(other == null) { + if (other == null) { return false; } - return other.contains(this.getMin()) - || other.contains(this.getMax()) - || this.contains(other.getMin()) - || this.contains(this.getMax()); + return other.contains(this.getMin()) || other.contains(this.getMax()) || this.contains( + other.getMin()) || this.contains(this.getMax()); } default boolean isOpen() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/common/QuarterUtils.java b/backend/src/main/java/com/bakdata/conquery/models/common/QuarterUtils.java index b04f8ff94e1..a9e3efa40aa 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/common/QuarterUtils.java +++ b/backend/src/main/java/com/bakdata/conquery/models/common/QuarterUtils.java @@ -10,17 +10,23 @@ import com.bakdata.conquery.models.common.daterange.CDateRange; import lombok.experimental.UtilityClass; + @UtilityClass public final class QuarterUtils { - private static final TemporalAdjuster FIRST_MONTH_IN_QUARTER_ADJUSTER = - temporal -> temporal.with(ChronoField.MONTH_OF_YEAR, getFirstMonthOfQuarter(temporal.get(IsoFields.QUARTER_OF_YEAR)).getValue()); + private static final TemporalAdjuster FIRST_MONTH_IN_QUARTER_ADJUSTER = temporal -> temporal.with( + ChronoField.MONTH_OF_YEAR, + getFirstMonthOfQuarter(temporal.get(IsoFields.QUARTER_OF_YEAR)).getValue()); - private static final TemporalAdjuster FIRST_DAY_OF_QUARTER_ADJUSTER = - temporal -> TemporalAdjusters.firstDayOfMonth().adjustInto(firstMonthInQuarterAdjuster().adjustInto(temporal)); + private static final TemporalAdjuster FIRST_DAY_OF_QUARTER_ADJUSTER = temporal -> TemporalAdjusters + .firstDayOfMonth() + .adjustInto( + firstMonthInQuarterAdjuster().adjustInto(temporal)); - private static final TemporalAdjuster LAST_DAY_OF_QUARTER_ADJUSTER = - temporal -> (TemporalAdjusters.firstDayOfMonth().adjustInto(nextQuarterAdjuster().adjustInto(temporal))).minus(1, ChronoUnit.DAYS); + private static final TemporalAdjuster LAST_DAY_OF_QUARTER_ADJUSTER = temporal -> (TemporalAdjusters + .firstDayOfMonth() + .adjustInto( + nextQuarterAdjuster().adjustInto(temporal))).minus(1, ChronoUnit.DAYS); public static Month getFirstMonthOfQuarter(int quarter) { @@ -37,19 +43,19 @@ public static LocalDate getFirstDayOfQuarter(int year, int quarter) { Month month = getFirstMonthOfQuarter(quarter); return LocalDate.of(year, month, 1); } - + public static LocalDate getFirstDayOfQuarter(int epochDay) { return LocalDate.ofEpochDay(epochDay).with(FIRST_DAY_OF_QUARTER_ADJUSTER); } - + public static LocalDate getFirstDayOfQuarter(LocalDate date) { return date.with(FIRST_DAY_OF_QUARTER_ADJUSTER); } - + public static LocalDate getLastDayOfQuarter(int epochDay) { return LocalDate.ofEpochDay(epochDay).with(LAST_DAY_OF_QUARTER_ADJUSTER); } - + public static LocalDate getLastDayOfQuarter(LocalDate date) { return date.with(LAST_DAY_OF_QUARTER_ADJUSTER); } @@ -63,7 +69,7 @@ public static CDateRange fromQuarter(int year, int quarter) { LocalDate end = getLastDayOfQuarter(year, quarter); return CDateRange.of(start, end); } - + public static CDateRange fromDate(LocalDate date) { return fromQuarter(date.getYear(), date.get(IsoFields.QUARTER_OF_YEAR)); } @@ -72,9 +78,13 @@ public static TemporalAdjuster nextQuarterAdjuster() { return temporal -> { if (temporal.get(IsoFields.QUARTER_OF_YEAR) == 4) { return temporal.with(ChronoField.MONTH_OF_YEAR, 1) - .with(ChronoField.YEAR, (temporal).get(ChronoField.YEAR) + 1); + .with( + ChronoField.YEAR, + (temporal).get(ChronoField.YEAR) + 1); } - return temporal.with(ChronoField.MONTH_OF_YEAR, getFirstMonthOfQuarter(temporal.get(IsoFields.QUARTER_OF_YEAR) + 1).getValue()); + return temporal.with( + ChronoField.MONTH_OF_YEAR, + getFirstMonthOfQuarter(temporal.get(IsoFields.QUARTER_OF_YEAR) + 1).getValue()); }; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/common/Range.java b/backend/src/main/java/com/bakdata/conquery/models/common/Range.java index c085c4aec20..66093928caf 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/common/Range.java +++ b/backend/src/main/java/com/bakdata/conquery/models/common/Range.java @@ -167,8 +167,9 @@ public IntegerRange(Integer min, Integer max) { } public static IntegerRange fromNumberRange(IRange orig) { - return new Range.IntegerRange(Optional.ofNullable(orig.getMin()).map(Number::intValue).orElse(null), - Optional.ofNullable(orig.getMax()).map(Number::intValue).orElse(null) + return new Range.IntegerRange( + Optional.ofNullable(orig.getMin()).map(Number::intValue).orElse(null), + Optional.ofNullable(orig.getMax()).map(Number::intValue).orElse(null) ); } @@ -198,8 +199,9 @@ public LongRange(Long min, Long max) { } public static LongRange fromNumberRange(IRange orig) { - return new Range.LongRange(Optional.ofNullable(orig.getMin()).map(Number::longValue).orElse(null), - Optional.ofNullable(orig.getMax()).map(Number::longValue).orElse(null) + return new Range.LongRange( + Optional.ofNullable(orig.getMin()).map(Number::longValue).orElse(null), + Optional.ofNullable(orig.getMax()).map(Number::longValue).orElse(null) ); } @@ -229,8 +231,9 @@ public FloatRange(Float min, Float max) { } public static FloatRange fromNumberRange(IRange orig) { - return new Range.FloatRange(Optional.ofNullable(orig.getMin()).map(Number::floatValue).orElse(null), - Optional.ofNullable(orig.getMax()).map(Number::floatValue).orElse(null) + return new Range.FloatRange( + Optional.ofNullable(orig.getMin()).map(Number::floatValue).orElse(null), + Optional.ofNullable(orig.getMax()).map(Number::floatValue).orElse(null) ); } @@ -263,8 +266,9 @@ public DoubleRange(Double min, Double max) { } public static DoubleRange fromNumberRange(IRange orig) { - return new Range.DoubleRange(Optional.ofNullable(orig.getMin()).map(Number::doubleValue).orElse(null), - Optional.ofNullable(orig.getMax()).map(Number::doubleValue).orElse(null) + return new Range.DoubleRange( + Optional.ofNullable(orig.getMin()).map(Number::doubleValue).orElse(null), + Optional.ofNullable(orig.getMax()).map(Number::doubleValue).orElse(null) ); } @@ -296,14 +300,18 @@ public MoneyRange(BigDecimal min, BigDecimal max) { super(min, max); } - public static MoneyRange fromNumberRange(IRange orig, FrontendConfig.CurrencyConfig currency) { + public static MoneyRange fromNumberRange( + IRange orig, + FrontendConfig.CurrencyConfig currency) { BigDecimal mappedMin = Optional.ofNullable(orig.getMin()) - .map(val -> new BigDecimal(val.longValue()).movePointLeft(currency.getDecimalScale())) - .orElse(null); + .map( + val -> new BigDecimal(val.longValue()).movePointLeft(currency.getDecimalScale())) + .orElse(null); BigDecimal mappedMax = Optional.ofNullable(orig.getMax()) - .map(val -> new BigDecimal(val.longValue()).movePointLeft(currency.getDecimalScale())) - .orElse(null); + .map( + val -> new BigDecimal(val.longValue()).movePointLeft(currency.getDecimalScale())) + .orElse(null); return new Range.MoneyRange(mappedMin, mappedMax); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/common/daterange/CDateRange.java b/backend/src/main/java/com/bakdata/conquery/models/common/daterange/CDateRange.java index 11bf0637969..e8f4af96b84 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/common/daterange/CDateRange.java +++ b/backend/src/main/java/com/bakdata/conquery/models/common/daterange/CDateRange.java @@ -33,7 +33,7 @@ private CDateRange(int min, int max) { if (min > max) { throw new IllegalArgumentException( - String.format("Min(%s) is not less than max(%s)", CDate.toLocalDate(min), CDate.toLocalDate(max))); + String.format("Min(%s) is not less than max(%s)", CDate.toLocalDate(min), CDate.toLocalDate(max))); } } @@ -184,7 +184,7 @@ public CDateRange intersection(CDateRange other) { @JsonValue public int[] asArray() { - return new int[] { getMinValue(), getMaxValue() }; + return new int[]{getMinValue(), getMaxValue()}; } @Override @@ -209,7 +209,7 @@ public CDateRange span(CDateRange other) { * @return A new closed span. */ public CDateRange spanClosed(CDateRange other) { - if(other == null){ + if (other == null) { return this; } @@ -218,19 +218,16 @@ public CDateRange spanClosed(CDateRange other) { // Initialize with the lowest know min from this if (hasLowerBound()) { min = getMinValue(); - } - else if(hasUpperBound()){ + } else if (hasUpperBound()) { min = getMaxValue(); - } - else { + } else { min = POSITIVE_INFINITY; } // then compare with others known min - if all fails, set it to MIN_VALUE if (other.hasLowerBound()) { min = Math.min(min, other.getMinValue()); - } - else if (other.hasUpperBound()){ + } else if (other.hasUpperBound()) { min = Math.min(min, other.getMaxValue()); } @@ -242,19 +239,16 @@ else if (other.hasUpperBound()){ int max; { if (hasUpperBound()) { - max = getMaxValue(); - } - else if (hasLowerBound()) { + max = getMaxValue(); + } else if (hasLowerBound()) { max = getMinValue(); - } - else { + } else { max = NEGATIVE_INFINITY; } - if(other.hasUpperBound()){ - max = Math.max(max,other.getMaxValue()); - } - else if(other.hasLowerBound()){ + if (other.hasUpperBound()) { + max = Math.max(max, other.getMaxValue()); + } else if (other.hasLowerBound()) { max = Math.max(max, other.getMinValue()); } @@ -264,11 +258,11 @@ else if(other.hasLowerBound()){ } - if(min == NEGATIVE_INFINITY && max != POSITIVE_INFINITY){ + if (min == NEGATIVE_INFINITY && max != POSITIVE_INFINITY) { min = max; } - if(max == POSITIVE_INFINITY && min != NEGATIVE_INFINITY){ + if (max == POSITIVE_INFINITY && min != NEGATIVE_INFINITY) { max = min; } @@ -321,11 +315,7 @@ public boolean intersects(CDateRange other) { return false; } - return !( - getMinValue() > other.getMaxValue() - || - getMaxValue() < other.getMinValue() - ); + return !(getMinValue() > other.getMaxValue() || getMaxValue() < other.getMinValue()); } public boolean encloses(CDateRange other) { @@ -333,9 +323,7 @@ public boolean encloses(CDateRange other) { return false; } - return - getMaxValue() >= other.getMaxValue() - && getMinValue() <= other.getMinValue(); + return getMaxValue() >= other.getMaxValue() && getMinValue() <= other.getMinValue(); } /** @@ -368,7 +356,7 @@ public Range toSimpleRange() { * @return The years as date ranges, from the first date in range to the last in ascending order. */ public List getCoveredYears() { - if(isOpen()){ + if (isOpen()) { // TODO: 22.04.2020 throw exceptiopn? return Collections.emptyList(); } @@ -376,7 +364,7 @@ public List getCoveredYears() { final int startYear = getMin().getYear(); final int endYear = getMax().getYear(); - if(startYear == endYear) { + if (startYear == endYear) { return List.of(this); } // Range covers multiple years @@ -386,12 +374,13 @@ public List getCoveredYears() { ranges.add(CDateRange.of(getMin(), LocalDate.of(startYear, 12, 31))); // Years in between - if(endYear-startYear > 1) { - ranges.addAll(IntStream - .rangeClosed(startYear+1, endYear-1) - // Create date range with first days of year and the last day - .mapToObj(year -> CDateRange.of(LocalDate.ofYearDay(year, 1), LocalDate.of(year, 12, 31))) - .collect(Collectors.toList())); + if (endYear - startYear > 1) { + ranges.addAll( + IntStream.rangeClosed(startYear + 1, endYear - 1) + // Create date range with first days of year and the last day + .mapToObj(year -> CDateRange.of(LocalDate.ofYearDay(year, 1), LocalDate.of(year, 12, 31))) + .collect( + Collectors.toList())); } // Last year end with this range ranges.add(CDateRange.of(LocalDate.of(endYear, 1, 1), getMax())); @@ -405,13 +394,13 @@ public List getCoveredYears() { * last in ascending order. */ public List getCoveredQuarters() { - if(isOpen()){ + if (isOpen()) { // TODO: 22.04.2020 throw exceptiopn? return Collections.emptyList(); } // If dateRange is shorter than a quarter, only add that first quarter. - if(QuarterUtils.getFirstDayOfQuarter(getMin()).isEqual(QuarterUtils.getFirstDayOfQuarter(getMax()))){ + if (QuarterUtils.getFirstDayOfQuarter(getMin()).isEqual(QuarterUtils.getFirstDayOfQuarter(getMax()))) { return List.of(this); } final List ranges = new ArrayList<>(); @@ -423,12 +412,12 @@ public List getCoveredQuarters() { ranges.add(start); LocalDate nextQuarterDate = getMin().plus(1, IsoFields.QUARTER_YEARS); - while(nextQuarterDate.isBefore(end.getMin())) { + while (nextQuarterDate.isBefore(end.getMin())) { ranges.add(QuarterUtils.fromDate(nextQuarterDate)); nextQuarterDate = nextQuarterDate.plus(1, IsoFields.QUARTER_YEARS); } // Don't add the end if it's the same quarter as start - if(!start.equals(end)) { + if (!start.equals(end)) { // Last year ends with this range ranges.add(end); } @@ -443,19 +432,19 @@ public List getCoveredQuarters() { * last in ascending order. */ public List getCoveredDays() { - if(isOpen()){ + if (isOpen()) { return Collections.emptyList(); } final List ranges = new ArrayList<>(); - for(int i = getMinValue(); i <= getMaxValue(); i++) { + for (int i = getMinValue(); i <= getMaxValue(); i++) { ranges.add(CDateRange.exactly(i)); } return ranges; } public boolean isSingleQuarter() { - if(isOpen()){ + if (isOpen()) { return false; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/APIConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/APIConfig.java index 8a0840278d9..5ce7295f618 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/APIConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/APIConfig.java @@ -3,9 +3,10 @@ import lombok.Getter; import lombok.Setter; -@Getter @Setter +@Getter +@Setter public class APIConfig { - + private boolean allowCORSRequests = false; private boolean caching = true; -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ArrowConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ArrowConfig.java index 6e4ffed3b42..61192a949f5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ArrowConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ArrowConfig.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.models.config; import jakarta.validation.constraints.Min; + import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ArrowResultProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/ArrowResultProvider.java index 4f043e0ddf1..414f2f77b2d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ArrowResultProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ArrowResultProvider.java @@ -1,13 +1,13 @@ package com.bakdata.conquery.models.config; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; +import jakarta.ws.rs.core.UriBuilder; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.util.Collection; import java.util.Collections; import java.util.List; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotNull; -import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.commands.ManagerNode; @@ -31,8 +31,10 @@ public class ArrowResultProvider implements ResultRendererProvider { private ArrowConfig config = new ArrowConfig(); @Override - public Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) - throws MalformedURLException, URISyntaxException { + public Collection generateResultURLs( + ManagedExecution exec, + UriBuilder uriBuilder, + boolean allProviders) throws MalformedURLException, URISyntaxException { if (!(exec instanceof SingleTableResult)) { return Collections.emptyList(); } @@ -42,9 +44,16 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuil } return List.of( - new ResultAsset("Arrow File", ResultArrowResource.getFileDownloadURL(uriBuilder.clone(), (ManagedExecution & SingleTableResult) exec).toURI()), - new ResultAsset("Arrow Stream", ResultArrowResource.getStreamDownloadURL(uriBuilder.clone(), (ManagedExecution & SingleTableResult) exec) - .toURI()) + new ResultAsset( + "Arrow File", + ResultArrowResource.getFileDownloadURL( + uriBuilder.clone(), + (ManagedExecution & SingleTableResult) exec).toURI()), + new ResultAsset( + "Arrow Stream", + ResultArrowResource.getStreamDownloadURL( + uriBuilder.clone(), + (ManagedExecution & SingleTableResult) exec).toURI()) ); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/CSVConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/CSVConfig.java index 3337b670899..1c7238e5bd2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/CSVConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/CSVConfig.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.config; +import jakarta.validation.constraints.NotNull; import java.io.OutputStream; import java.io.Writer; import java.nio.charset.Charset; @@ -10,7 +11,6 @@ import com.univocity.parsers.csv.CsvParserSettings; import com.univocity.parsers.csv.CsvWriter; import com.univocity.parsers.csv.CsvWriterSettings; -import jakarta.validation.constraints.NotNull; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; @@ -20,14 +20,19 @@ import org.hibernate.validator.constraints.Length; /** - * Holds the necessary information to configure CSV parsers and writers. + * Holds the necessary information to configure CSV parsers and writers. */ -@Getter @Setter @With @AllArgsConstructor @NoArgsConstructor +@Getter +@Setter +@With +@AllArgsConstructor +@NoArgsConstructor public class CSVConfig { private char escape = '\\'; private char comment = '\0'; private char delimeter = ','; - @Length(min=1, max=2) @NotNull + @Length(min = 1, max = 2) + @NotNull private String lineSeparator = "\n"; private char quote = '"'; @NotNull @@ -35,7 +40,7 @@ public class CSVConfig { private boolean skipHeader = false; private boolean parseHeaders = true; private int maxColumns = 1_000_000; // This should be sufficiently large. - + /** * Helper method to generate parser settings from the provided options in this class. * @return Setting object that can be passed into a {@link CsvParser}. @@ -59,7 +64,7 @@ private CsvWriterSettings createCsvWriterSettings() { settings.setFormat(createCsvFormat()); return settings; } - + /** * Helper method to generate format settings from the provided options in this class. * @return Format object that can be passed into {@link CsvWriterSettings} and {@link CsvParserSettings}. @@ -106,4 +111,4 @@ public CsvWriter createWriter(@NonNull OutputStream out) { return new CsvWriter(out, createCsvWriterSettings()); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ClusterConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ClusterConfig.java index 2082890ef21..032ff8708fe 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ClusterConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ClusterConfig.java @@ -1,5 +1,11 @@ package com.bakdata.conquery.models.config; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; + import com.bakdata.conquery.io.mina.MdcFilter; import com.bakdata.conquery.io.mina.jackson.AsyncJacksonProtocolFilter; import com.bakdata.conquery.models.messages.namespaces.specific.CollectColumnValuesMessage; @@ -10,9 +16,6 @@ import io.dropwizard.util.Duration; import io.dropwizard.validation.DataSizeRange; import io.dropwizard.validation.PortRange; -import jakarta.validation.Valid; -import jakarta.validation.constraints.Min; -import jakarta.validation.constraints.NotNull; import lombok.Getter; import lombok.Setter; import org.apache.mina.core.filterchain.IoFilter; @@ -21,10 +24,6 @@ import org.apache.mina.transport.socket.nio.NioSocketAcceptor; import org.apache.mina.transport.socket.nio.NioSocketConnector; -import java.io.IOException; -import java.net.InetAddress; -import java.net.InetSocketAddress; - @Getter @Setter public class ClusterConfig extends Configuration { @@ -81,7 +80,10 @@ public NioSocketConnector getClusterConnector(ObjectMapper om, IoHandler ioHandl } @JsonIgnore - public NioSocketAcceptor getClusterAcceptor(ObjectMapper om, IoHandler ioHandler, String mdcLocation) throws IOException { + public NioSocketAcceptor getClusterAcceptor( + ObjectMapper om, + IoHandler ioHandler, + String mdcLocation) throws IOException { final NioSocketAcceptor acceptor = new NioSocketAcceptor(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java index a708564a402..9ec4dcfdeb3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.config; +import jakarta.validation.constraints.NotEmpty; import java.util.Collections; import java.util.Locale; import java.util.Map; @@ -12,7 +13,6 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonView; import com.google.common.base.Strings; -import jakarta.validation.constraints.NotEmpty; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Getter; diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java index 914e283126b..72e4b9418d6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.config; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; import java.time.LocalDate; import java.util.ArrayList; import java.util.HashSet; @@ -7,8 +9,6 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Stream; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.io.jackson.Injectable; import com.bakdata.conquery.io.jackson.MutableInjectableValues; @@ -63,11 +63,11 @@ public class ConqueryConfig extends Configuration implements Injectable { @Valid @NotNull private List resultProviders = List.of( - new ExcelResultProvider(), - new CsvResultProvider(), - new ArrowResultProvider(), - new ParquetResultProvider(), - new ExternalResultProvider() + new ExcelResultProvider(), + new CsvResultProvider(), + new ArrowResultProvider(), + new ParquetResultProvider(), + new ExternalResultProvider() ); @Valid @NotNull @@ -137,7 +137,9 @@ boolean isPluginUnique() { // Normal plugins only exist once per class Set> singleInstanceClasses = new HashSet<>(); // MultiInstance plugins have a distinct id among their class - Multimap, String> multiInstanceClasses = MultimapBuilder.hashKeys().hashSetValues().build(); + Multimap, String> multiInstanceClasses = MultimapBuilder.hashKeys() + .hashSetValues() + .build(); for (PluginConfig plugin : plugins) { if (plugin instanceof MultiInstancePlugin mu) { @@ -156,16 +158,18 @@ boolean isPluginUnique() { public Optional getPluginConfig(Class type) { return plugins.stream() - .filter(c -> type.isAssignableFrom(c.getClass())) - .map(type::cast) - .collect(MoreCollectors.toOptional()); + .filter(c -> type.isAssignableFrom(c.getClass())) + .map(type::cast) + .collect( + MoreCollectors.toOptional()); } public Stream getPluginConfigs(Class type) { return plugins.stream() - .filter(c -> type.isAssignableFrom(c.getClass())) - .filter(c -> MultiInstancePlugin.class.isAssignableFrom(c.getClass())) - .map(type::cast); + .filter(c -> type.isAssignableFrom(c.getClass())) + .filter( + c -> MultiInstancePlugin.class.isAssignableFrom(c.getClass())) + .map(type::cast); } public ObjectMapper configureObjectMapper(ObjectMapper objectMapper) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryMetricsConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryMetricsConfig.java index 219ffec2eb2..6811f945f8f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryMetricsConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryMetricsConfig.java @@ -4,7 +4,8 @@ import lombok.Getter; import lombok.Setter; -@Getter @Setter +@Getter +@Setter public class ConqueryMetricsConfig { private Duration userActiveDuration = Duration.hours(1); private int groupTrackingMinSize = 3; diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/CsvResultProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/CsvResultProvider.java index 005a89c2a56..43f138dfbbd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/CsvResultProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/CsvResultProvider.java @@ -1,11 +1,11 @@ package com.bakdata.conquery.models.config; +import jakarta.ws.rs.core.UriBuilder; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.util.Collection; import java.util.Collections; import java.util.List; -import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.commands.ManagerNode; @@ -26,8 +26,10 @@ public class CsvResultProvider implements ResultRendererProvider { private boolean hidden = false; - public Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) - throws MalformedURLException, URISyntaxException { + public Collection generateResultURLs( + ManagedExecution exec, + UriBuilder uriBuilder, + boolean allProviders) throws MalformedURLException, URISyntaxException { if (!(exec instanceof SingleTableResult)) { return Collections.emptyList(); } @@ -36,7 +38,10 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuil return Collections.emptyList(); } - return List.of(new ResultAsset("CSV", ResultCsvResource.getDownloadURL(uriBuilder, (ManagedExecution & SingleTableResult) exec).toURI())); + return List.of( + new ResultAsset( + "CSV", + ResultCsvResource.getDownloadURL(uriBuilder, (ManagedExecution & SingleTableResult) exec).toURI())); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConnectionConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConnectionConfig.java index 2c5cfb3fe62..d655f610dec 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConnectionConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConnectionConfig.java @@ -1,11 +1,12 @@ package com.bakdata.conquery.models.config; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotNull; + import com.codahale.metrics.health.HealthCheckRegistry; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import io.dropwizard.util.Duration; -import jakarta.validation.constraints.Min; -import jakarta.validation.constraints.NotNull; import lombok.*; import lombok.extern.jackson.Jacksonized; import lombok.extern.slf4j.Slf4j; diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java b/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java index 957fa075f6c..f9080889a87 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java @@ -15,8 +15,7 @@ @Getter public enum Dialect { - CLICKHOUSE(new ClickhouseDialectBundle()), - HANA(new HanaDialectBundle()); + CLICKHOUSE(new ClickhouseDialectBundle()), HANA(new HanaDialectBundle()); private final DialectBundle dialectBundle; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelConfig.java index bdf0cc42d41..3cd3ed9fbfc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelConfig.java @@ -1,10 +1,10 @@ package com.bakdata.conquery.models.config; -import java.util.Collections; -import java.util.Map; import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotNull; +import java.util.Collections; +import java.util.Map; import com.bakdata.conquery.models.query.PrintSettings; import com.google.common.collect.ImmutableMap; @@ -29,11 +29,15 @@ public class ExcelConfig { public static final String INTEGER_STYLE = "integer"; private static final Map FALLBACK_STYLES = Map.of( - BASIC_STYLE, new CellStyler(), - // \u00A0 is the non breakable space - CURRENCY_STYLE_PREFIX + "EUR", new CellStyler().withDataFormatString("#,##0.00\u00A0€"), - NUMERIC_STYLE, new CellStyler().withDataFormatString("#,##0.00"), - INTEGER_STYLE, new CellStyler().withDataFormatString("#,##0") + BASIC_STYLE, + new CellStyler(), + // \u00A0 is the non breakable space + CURRENCY_STYLE_PREFIX + "EUR", + new CellStyler().withDataFormatString("#,##0.00\u00A0€"), + NUMERIC_STYLE, + new CellStyler().withDataFormatString("#,##0.00"), + INTEGER_STYLE, + new CellStyler().withDataFormatString("#,##0") ); /** @@ -62,7 +66,7 @@ INTEGER_STYLE, new CellStyler().withDataFormatString("#,##0") private int lastRowToAutosize = 30; - public ImmutableMap generateStyles(SXSSFWorkbook workbook, PrintSettings settings){ + public ImmutableMap generateStyles(SXSSFWorkbook workbook, PrintSettings settings) { ImmutableMap.Builder styles = ImmutableMap.builder(); // Add localized DateCell style @@ -76,7 +80,7 @@ public ImmutableMap generateStyles(SXSSFWorkbook workbook, Pr // Add missing basic styles for (String s : FALLBACK_STYLES.keySet()) { - if(this.styles.containsKey(s)){ + if (this.styles.containsKey(s)) { continue; } styles.put(s, FALLBACK_STYLES.get(s).generateStyle(workbook)); @@ -105,14 +109,14 @@ private static class CellStyler { private CellStyle generateStyle(SXSSFWorkbook workbook) { DataFormat dataFormat = workbook.createDataFormat(); CellStyle style = workbook.createCellStyle(); - if (fillPattern != null){ + if (fillPattern != null) { style.setFillPattern(fillPattern); } if (foregroundColors != null) { style.setFillForegroundColor(foregroundColors); } - if (dataFormatString != null){ + if (dataFormatString != null) { style.setDataFormat(dataFormat.getFormat(dataFormatString)); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java index cf94d9953e0..96459b780d7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java @@ -1,5 +1,9 @@ package com.bakdata.conquery.models.config; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.UriBuilder; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; @@ -7,10 +11,6 @@ import java.util.Collections; import java.util.List; import java.util.OptionalLong; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotNull; -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.commands.ManagerNode; @@ -34,7 +34,9 @@ public class ExcelResultProvider implements ResultRendererProvider { // Media type according to https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types - public static final MediaType MEDIA_TYPE = new MediaType("application", "vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + public static final MediaType MEDIA_TYPE = new MediaType( + "application", + "vnd.openxmlformats-officedocument.spreadsheetml.sheet"); private boolean hidden = false; @@ -47,7 +49,10 @@ public class ExcelResultProvider implements ResultRendererProvider { @Override @SneakyThrows(MalformedURLException.class) - public Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) throws URISyntaxException { + public Collection generateResultURLs( + ManagedExecution exec, + UriBuilder uriBuilder, + boolean allProviders) throws URISyntaxException { // We only support/produce xlsx files with one sheet for now if (!(exec instanceof SingleTableResult singleExecution)) { log.trace("Execution result is not a single table"); @@ -73,12 +78,15 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuil final int maxRowCount = SpreadsheetVersion.EXCEL2007.getMaxRows(); if (rowCount.getAsLong() + 1 /* header row*/ > maxRowCount) { - log.trace("Row count is too high for XLSX format (is: {}, max: {}). Not producing a result URL", rowCount, maxRowCount); + log.trace( + "Row count is too high for XLSX format (is: {}, max: {}). Not producing a result URL", + rowCount, + maxRowCount); return Collections.emptyList(); } - if(singleExecution.getResultInfos() == null){ + if (singleExecution.getResultInfos() == null) { return Collections.emptyList(); } @@ -89,12 +97,17 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuil final int maxColumnCount = SpreadsheetVersion.EXCEL2007.getMaxColumns(); if (columnCount > maxColumnCount) { - log.trace("Column count is too high for XLSX format (is: {}, max: {}). Not producing a result URL", columnCount, maxColumnCount); + log.trace( + "Column count is too high for XLSX format (is: {}, max: {}). Not producing a result URL", + columnCount, + maxColumnCount); return Collections.emptyList(); } - final URL resultUrl = ResultExcelResource.getDownloadURL(uriBuilder, (ManagedExecution & SingleTableResult) exec); + final URL resultUrl = ResultExcelResource.getDownloadURL( + uriBuilder, + (ManagedExecution & SingleTableResult) exec); log.trace("Generated URL: {}", resultUrl); return List.of(new ResultAsset("XLSX", resultUrl.toURI())); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ExternalResultProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/ExternalResultProvider.java index 2d5dea440c2..b7c27bd0f10 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ExternalResultProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ExternalResultProvider.java @@ -1,8 +1,8 @@ package com.bakdata.conquery.models.config; +import jakarta.ws.rs.core.UriBuilder; import java.util.Collection; import java.util.Collections; -import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.commands.ManagerNode; @@ -27,7 +27,10 @@ public class ExternalResultProvider implements ResultRendererProvider { private boolean hidden = false; @Override - public Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) { + public Collection generateResultURLs( + ManagedExecution exec, + UriBuilder uriBuilder, + boolean allProviders) { if (!(exec instanceof ExternalExecution)) { return Collections.emptyList(); @@ -37,7 +40,10 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuil return Collections.emptyList(); } - return ((ExternalExecution) exec).getResultAssets().map(assetBuilder -> assetBuilder.apply(uriBuilder.clone())).toList(); + return ((ExternalExecution) exec).getResultAssets() + .map( + assetBuilder -> assetBuilder.apply(uriBuilder.clone())) + .toList(); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java index ea9436fba7f..3d0f1dc92dd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java @@ -1,5 +1,10 @@ package com.bakdata.conquery.models.config; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Pattern; +import jakarta.ws.rs.client.Client; import java.io.IOException; import java.net.URI; import java.net.URL; @@ -7,11 +12,6 @@ import java.util.HashSet; import java.util.Set; import javax.annotation.Nullable; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; -import jakarta.validation.constraints.Pattern; -import jakarta.ws.rs.client.Client; import com.bakdata.conquery.apiv1.forms.ExternalForm; import com.bakdata.conquery.apiv1.frontend.VersionContainer; @@ -111,7 +111,6 @@ public void initialize(ManagerNode managerNode) { client = conqueryClientBuilder.using(managerNode.getConfig().getJerseyClient()).build(getId()); - final ObjectMapper om = configureObjectMapper(managerNode.getEnvironment().getObjectMapper().copy()); client.register(new JacksonMessageBodyProvider(om)); @@ -122,7 +121,9 @@ public void initialize(ManagerNode managerNode) { // Register form configuration provider log.info("Registering frontend form config provider for '{}'", getId()); - managerNode.getFormScanner().registerFrontendFormConfigProvider(new FormConfigProvider(getId(), this::registerFormConfigs)); + managerNode.getFormScanner() + .registerFrontendFormConfigProvider( + new FormConfigProvider(getId(), this::registerFormConfigs)); } public static ObjectMapper configureObjectMapper(ObjectMapper om) { @@ -130,7 +131,18 @@ public static ObjectMapper configureObjectMapper(ObjectMapper om) { } public ExternalFormBackendApi createApi() { - return new ExternalFormBackendApi(client, baseURI, formConfigPath, postFormPath, statusTemplatePath, cancelTaskPath, healthCheckPath, versionPath, this::createAccessToken, conqueryApiUrl, getAuthentication()); + return new ExternalFormBackendApi( + client, + baseURI, + formConfigPath, + postFormPath, + statusTemplatePath, + cancelTaskPath, + healthCheckPath, + versionPath, + this::createAccessToken, + conqueryApiUrl, + getAuthentication()); } /** @@ -172,7 +184,11 @@ private String createAccessToken(User user) { * @param subType the type that will be the subtype of {@link ExternalForm} */ public static String createSubTypedId(String subType) { - return String.format("%s%s%s", ExternalForm.class.getAnnotation(CPSType.class).id(), CPSTypeIdResolver.SEPARATOR_SUB_TYPE, subType); + return String.format( + "%s%s%s", + ExternalForm.class.getAnnotation(CPSType.class).id(), + CPSTypeIdResolver.SEPARATOR_SUB_TYPE, + subType); } /** @@ -182,17 +198,16 @@ private void updateVersion(ExternalFormBackendApi externalApi) { try { final FormBackendVersion versionInfo = externalApi.getVersion(); - final VersionContainer - oldVersion = - VersionInfo.INSTANCE.setFormBackendVersion(new VersionContainer(getId(), versionInfo.version(), versionInfo.buildTime())); + final VersionContainer oldVersion = VersionInfo.INSTANCE.setFormBackendVersion( + new VersionContainer(getId(), versionInfo.version(), versionInfo.buildTime())); if (!versionInfo.version().equals(oldVersion.version())) { log.info("Form Backend '{}' versionInfo update: {} -> {}", getId(), oldVersion, versionInfo); } - } - catch (Exception e) { - log.warn("Unable to retrieve version from form backend '{}'. Enable trace logging for more info", getId(), (Exception) (log.isTraceEnabled() - ? e - : null)); + } catch (Exception e) { + log.warn( + "Unable to retrieve version from form backend '{}'. Enable trace logging for more info", + getId(), + (Exception) (log.isTraceEnabled() ? e : null)); VersionInfo.INSTANCE.setFormBackendVersion(new VersionContainer(getId(), null, null)); } @@ -212,8 +227,7 @@ public void close() { if (internalHttpClient != null) { internalHttpClient.getClient().close(); } - } - catch (IOException e) { + } catch (IOException e) { throw new RuntimeException(e); } } @@ -227,9 +241,10 @@ public User createServiceUser(User originalUser, Dataset dataset) { // We create a service user with elevated permissions (allow downloads, which a form backend relies on) so we don't // have to give the permission to the actual user. The service user will have a snapshot of the permissions of // the actual user and download permissions. - final User - serviceUser = - managerNode.getAuthController().flatCopyUser(originalUser, String.format("%s_%s", getClass().getSimpleName().toLowerCase(), getId())); + final User serviceUser = managerNode.getAuthController() + .flatCopyUser( + originalUser, + String.format("%s_%s", getClass().getSimpleName().toLowerCase(), getId())); // The user is able to read the dataset, ensure that the service user can download results serviceUser.addPermission(dataset.createPermission(Ability.DOWNLOAD.asSet())); @@ -251,10 +266,13 @@ public ConqueryClientBuilder(Environment environment) { } @Override - protected DropwizardApacheConnector createDropwizardApacheConnector(ConfiguredCloseableHttpClient configuredClient) { + protected DropwizardApacheConnector createDropwizardApacheConnector( + ConfiguredCloseableHttpClient configuredClient) { httpClient = configuredClient; - return new DropwizardApacheConnector(configuredClient.getClient(), configuredClient.getDefaultRequestConfig(), - true); + return new DropwizardApacheConnector( + configuredClient.getClient(), + configuredClient.getDefaultRequestConfig(), + true); } } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java index 90f678dcd14..b996dd96ad3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java @@ -1,13 +1,12 @@ package com.bakdata.conquery.models.config; -import java.net.URI; -import java.net.URL; - -import javax.annotation.Nullable; import jakarta.validation.Valid; import jakarta.validation.constraints.Email; import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotNull; +import java.net.URI; +import java.net.URL; +import javax.annotation.Nullable; import com.bakdata.conquery.models.forms.frontendconfiguration.FormScanner; import com.fasterxml.jackson.annotation.JsonAlias; @@ -77,7 +76,7 @@ public class FrontendConfig { @ValidationMethod(message = "Percentiles must be concrete and within 0 - 100") @JsonIgnore public boolean isValidPercentiles() { - if(!visualisationPercentiles.hasLowerBound() || !visualisationPercentiles.hasUpperBound()){ + if (!visualisationPercentiles.hasLowerBound() || !visualisationPercentiles.hasUpperBound()) { return false; } @@ -102,4 +101,4 @@ public static class CurrencyConfig { } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java index 28e0e7beb56..c894e0916aa 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.config; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -22,8 +24,6 @@ import com.google.common.base.Functions; import com.google.common.collect.MoreCollectors; import io.dropwizard.validation.ValidationMethod; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotEmpty; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Getter; @@ -56,27 +56,31 @@ public class IdColumnConfig { @NotEmpty @Valid private List ids = List.of( - ColumnConfig.builder() - .name("ID") - .field("pid") - .label(Map.of(Locale.ROOT, "result")) - .primaryId(true) - .print(true) - .build() + ColumnConfig.builder() + .name("ID") + .field("pid") + .label(Map.of(Locale.ROOT, "result")) + .primaryId(true) + .print( + true) + .build() ); @JsonIgnore @Setter(AccessLevel.NONE) @Getter(lazy = true, value = AccessLevel.PUBLIC) - private final Map idMappers = ids.stream().collect(Collectors.toMap(ColumnConfig::getName, Functions.identity())); + private final Map idMappers = ids.stream() + .collect( + Collectors.toMap(ColumnConfig::getName, Functions.identity())); @JsonIgnore public ColumnConfig findPrimaryIdColumn() { return ids.stream() - .filter(ColumnConfig::isPrimaryId) - .findFirst() - .orElseThrow(() -> new IllegalStateException("Requiring at least 1 primary key column in IdColumnConfig")); + .filter(ColumnConfig::isPrimaryId) + .findFirst() + .orElseThrow( + () -> new IllegalStateException("Requiring at least 1 primary key column in IdColumnConfig")); } @ValidationMethod(message = "Duplicate Claims for Mapping Columns.") @@ -110,10 +114,7 @@ public boolean isIdColsHaveMapping() { @ValidationMethod(message = "Must have exactly one Column for Pseudomization.") @JsonIgnore public boolean isExactlyOnePseudo() { - return ids.stream() - .filter(conf -> conf.getField() != null) - .filter(ColumnConfig::isPrimaryId) - .count() == 1; + return ids.stream().filter(conf -> conf.getField() != null).filter(ColumnConfig::isPrimaryId).count() == 1; } @@ -134,11 +135,16 @@ public String userColumnName(PrintSettings printSettings) { // Get the label for the locale, // fall back to any label if there is exactly one defined, // then fall back to the field name. - return Objects.requireNonNullElse(labels.getOrDefault( + return Objects.requireNonNullElse( + labels.getOrDefault( printSettings.getLocale(), // fall backs - labels.size() == 1 ? labels.values().stream().collect(MoreCollectors.onlyElement()) : col.getField() - ), col.getField()); + labels.size() == 1 ? labels.values() + .stream() + .collect( + MoreCollectors.onlyElement()) : col.getField() + ), + col.getField()); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/IndexConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/IndexConfig.java index edb2110b202..d8b2b60009f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/IndexConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/IndexConfig.java @@ -5,12 +5,10 @@ import java.net.URL; import javax.annotation.Nullable; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import jakarta.validation.constraints.NotNull; - import com.bakdata.conquery.apiv1.FilterTemplate; import com.bakdata.conquery.models.index.IndexKey; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import io.dropwizard.validation.ValidationMethod; import lombok.Data; import lombok.extern.slf4j.Slf4j; @@ -18,7 +16,7 @@ @Data @Slf4j @JsonIgnoreProperties({"emptyLabel"}) -public class IndexConfig { +public class IndexConfig { /** * Base url under which reference files are access if the url in @@ -41,8 +39,7 @@ public boolean isValidUrl() { // We just try to convert it to an url and discard the return value URL _ignore = baseUrl.toURL(); return true; - } - catch (MalformedURLException e) { + } catch (MalformedURLException e) { log.error("URL validation error.", e); return false; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/JsonDescriptionResultProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/JsonDescriptionResultProvider.java index 139cedf7a5a..a30afdc6999 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/JsonDescriptionResultProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/JsonDescriptionResultProvider.java @@ -1,11 +1,11 @@ package com.bakdata.conquery.models.config; +import jakarta.ws.rs.core.UriBuilder; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.util.Collection; import java.util.Collections; import java.util.List; -import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.commands.ManagerNode; @@ -26,14 +26,17 @@ public class JsonDescriptionResultProvider implements ResultRendererProvider { private boolean hidden = false; @Override - public Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) - throws MalformedURLException, URISyntaxException { + public Collection generateResultURLs( + ManagedExecution exec, + UriBuilder uriBuilder, + boolean allProviders) throws MalformedURLException, URISyntaxException { if (hidden && !allProviders) { return Collections.emptyList(); } - return List.of(new ResultAsset("JSON", ResultJsonDescriptionResource.getDownloadURL(uriBuilder, exec.getId()).toURI())); + return List.of( + new ResultAsset("JSON", ResultJsonDescriptionResource.getDownloadURL(uriBuilder, exec.getId()).toURI())); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java index 15f222e30f9..b8b2d234e49 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java @@ -1,5 +1,8 @@ package com.bakdata.conquery.models.config; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Size; import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -7,10 +10,6 @@ import java.util.Set; import java.util.StringJoiner; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; -import jakarta.validation.constraints.Size; - import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.exceptions.ParsingException; @@ -36,8 +35,10 @@ public class LocaleConfig { */ @NotNull private Map dateFormatMapping = Map.of( - Locale.GERMAN, "dd.MM.yyyy", - Locale.ROOT, "yyyy-MM-dd" + Locale.GERMAN, + "dd.MM.yyyy", + Locale.ROOT, + "yyyy-MM-dd" ); @@ -46,7 +47,7 @@ public class LocaleConfig { */ @NotNull private Set parsingDateFormats = Set.of( - "yyyyMMdd" + "yyyyMMdd" ); /** @@ -56,8 +57,10 @@ public class LocaleConfig { */ @NotEmpty private Map localeRangeStartEndSeparators = Map.of( - Locale.GERMAN, " - ", - Locale.ROOT, "/" + Locale.GERMAN, + " - ", + Locale.ROOT, + "/" ); /** @@ -73,10 +76,10 @@ public class LocaleConfig { @NotNull @NotEmpty private List listFormats = List.of( - new ListFormat("", " ; ", ""), - new ListFormat("", ", ", ""), - new ListFormat("{", ",", "}"), - new ListFormat("[", ",", "]") + new ListFormat("", " ; ", ""), + new ListFormat("", ", ", ""), + new ListFormat("{", ",", "}"), + new ListFormat("[", ",", "]") ); /** @@ -126,7 +129,7 @@ public CDateSet parse(String value, DateReader reader) { final CDateSet out = CDateSet.createEmpty(); // After stripping start and end, the trimmed string is empty - if(value.isEmpty()){ + if (value.isEmpty()) { return out; } @@ -159,9 +162,9 @@ public DateReader getDateReader() { final List rangeStartEndSeperators = new ArrayList<>(localeRangeStartEndSeparators.values()); rangeStartEndSeperators.addAll(parsingRangeStartEndSeparators); return new DateReader( - Sets.union(parsingDateFormats, Set.copyOf(dateFormatMapping.values())), - rangeStartEndSeperators, - listFormats + Sets.union(parsingDateFormats, Set.copyOf(dateFormatMapping.values())), + rangeStartEndSeperators, + listFormats ); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ManualConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ManualConfig.java index e9e719ab4e7..ba58715c318 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ManualConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ManualConfig.java @@ -1,14 +1,9 @@ package com.bakdata.conquery.models.config; -import java.net.MalformedURLException; import java.net.URI; import java.util.Collections; -import java.util.List; import java.util.Map; -import jakarta.validation.ConstraintValidator; -import jakarta.validation.ConstraintValidatorContext; - import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.util.validation.ManualURI; import lombok.Getter; diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/MinaConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/MinaConfig.java index fdeea772cef..213bd50f32a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/MinaConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/MinaConfig.java @@ -88,14 +88,11 @@ public void setIdleTime(IdleStatus status, int idleTime) { if (status == IdleStatus.BOTH_IDLE) { bothIdleTime = idleTime; - } - else if (status == IdleStatus.READER_IDLE) { + } else if (status == IdleStatus.READER_IDLE) { readerIdleTime = idleTime; - } - else if (status == IdleStatus.WRITER_IDLE) { + } else if (status == IdleStatus.WRITER_IDLE) { writerIdleTime = idleTime; - } - else { + } else { throw new IllegalArgumentException("Unknown idle status: " + status); } } @@ -172,4 +169,4 @@ public long getWriteTimeoutInMillis() { public long getThroughputCalculationIntervalInMillis() { return throughputCalculationInterval * 1000L; } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ParquetResultProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/ParquetResultProvider.java index 7456e861503..1a0caf4ebde 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ParquetResultProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ParquetResultProvider.java @@ -1,11 +1,11 @@ package com.bakdata.conquery.models.config; +import jakarta.ws.rs.core.UriBuilder; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.util.Collection; import java.util.Collections; import java.util.List; -import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.commands.ManagerNode; @@ -26,8 +26,10 @@ public class ParquetResultProvider implements ResultRendererProvider { private boolean hidden = true; @Override - public Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) - throws MalformedURLException, URISyntaxException { + public Collection generateResultURLs( + ManagedExecution exec, + UriBuilder uriBuilder, + boolean allProviders) throws MalformedURLException, URISyntaxException { if (!(exec instanceof SingleTableResult)) { return Collections.emptyList(); } @@ -37,7 +39,11 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuil } return List.of( - new ResultAsset("PARQUET", ResultParquetResource.getDownloadURL(uriBuilder.clone(), (ManagedExecution & SingleTableResult) exec).toURI()) + new ResultAsset( + "PARQUET", + ResultParquetResource.getDownloadURL( + uriBuilder.clone(), + (ManagedExecution & SingleTableResult) exec).toURI()) ); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ParserConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ParserConfig.java index b3477957825..d17fee413ed 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ParserConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ParserConfig.java @@ -1,29 +1,27 @@ package com.bakdata.conquery.models.config; -import com.bakdata.conquery.util.DateReader; -import lombok.Data; -import lombok.Setter; - import jakarta.validation.constraints.NotNull; - import java.util.*; +import lombok.Data; +import lombok.Setter; + @Data @Setter public class ParserConfig { - /** - * Minimum required double precision to switch from floats to doubles. - * If set to a positive value, preprocessing will select float values for columns with at least minPrecision ulp. - * - * @see com.bakdata.conquery.models.preproc.parser.specific.RealParser#decideType() - * @see Math#ulp(float) - */ - private double minPrecision = Double.MIN_VALUE; + /** + * Minimum required double precision to switch from floats to doubles. + * If set to a positive value, preprocessing will select float values for columns with at least minPrecision ulp. + * + * @see com.bakdata.conquery.models.preproc.parser.specific.RealParser#decideType() + * @see Math#ulp(float) + */ + private double minPrecision = Double.MIN_VALUE; - /** - * The currency type of currency values that are parsed and processed. For now there can only be one currency in - * an instance. - */ - @NotNull - private Currency currency = Currency.getInstance("EUR"); + /** + * The currency type of currency values that are parsed and processed. For now there can only be one currency in + * an instance. + */ + @NotNull + private Currency currency = Currency.getInstance("EUR"); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/PluginConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/PluginConfig.java index b50775420cd..e538ccf078c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/PluginConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/PluginConfig.java @@ -8,7 +8,9 @@ @CPSBase public interface PluginConfig { - default void initialize(ManagerNode managerNode){} + default void initialize(ManagerNode managerNode) { + } - default void close(){} + default void close() { + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/PreprocessingConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/PreprocessingConfig.java index 946db5afbfd..a4b8b2ee23d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/PreprocessingConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/PreprocessingConfig.java @@ -7,14 +7,17 @@ import lombok.Setter; import lombok.ToString; -@Getter @Setter @ToString +@Getter +@Setter +@ToString public class PreprocessingConfig { @Min(1) private int nThreads = Runtime.getRuntime().availableProcessors(); @Min(0) private int maximumPrintedErrors = 10; - @Min(0) @Max(1) + @Min(0) + @Max(1) private double faultyLineThreshold = 0.01d; private ParserConfig parsers = new ParserConfig(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/QueryConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/QueryConfig.java index 1c9d191db21..58a311f64f5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/QueryConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/QueryConfig.java @@ -1,9 +1,7 @@ package com.bakdata.conquery.models.config; import com.bakdata.conquery.util.validation.ValidCaffeineSpec; -import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.util.Duration; -import io.dropwizard.validation.ValidationMethod; import lombok.Getter; import lombok.Setter; import lombok.ToString; @@ -35,7 +33,7 @@ public class QueryConfig { /** * See {@link com.bakdata.conquery.models.query.ExecutionManager#executionInfosL1} for an explanation */ - @ValidCaffeineSpec(softValue=true) + @ValidCaffeineSpec(softValue = true) private String L2CacheSpec = "softValues"; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java index d646d8344a8..5bf50caefbe 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java @@ -1,7 +1,7 @@ package com.bakdata.conquery.models.config; -import java.util.Map; import jakarta.validation.Valid; +import java.util.Map; import com.bakdata.conquery.mode.local.ConnectionManager; import com.bakdata.conquery.mode.local.ManagedConnection; @@ -48,7 +48,11 @@ public ConnectionManager toConnectionManager(Environment environment) { ConnectionManager connectionManager = new ConnectionManager(); for (Map.Entry configEntry : getDatabaseConfigs().entrySet()) { - ManagedConnection managedConnection = new ManagedConnection(configEntry.getKey(), this, configEntry.getValue(), environment.healthChecks()); + ManagedConnection managedConnection = new ManagedConnection( + configEntry.getKey(), + this, + configEntry.getValue(), + environment.healthChecks()); environment.lifecycle().manage(managedConnection); connectionManager.addConnection(configEntry.getKey(), managedConnection); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/StandaloneConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/StandaloneConfig.java index de073c7308c..8e96c4fa488 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/StandaloneConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/StandaloneConfig.java @@ -3,7 +3,8 @@ import lombok.Getter; import lombok.Setter; -@Getter @Setter +@Getter +@Setter public class StandaloneConfig { private int numberOfShardNodes = 2; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java index 47ebafa3657..9314cf1826e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java @@ -47,7 +47,9 @@ public interface StoreFactory { // NamespacedStorage (Important for serdes communication between manager and shards) SingletonStore createDatasetStore(String pathName, ObjectMapper objectMapper); - IdentifiableStore createSecondaryIdDescriptionStore(String pathName, ObjectMapper objectMapper); + IdentifiableStore createSecondaryIdDescriptionStore( + String pathName, + ObjectMapper objectMapper); IdentifiableStore createTableStore(String pathName, ObjectMapper objectMapper); @@ -80,7 +82,9 @@ public interface StoreFactory { IdentifiableStore createGroupStore(String pathName, ObjectMapper objectMapper); - IdentifiableStore createInternToExternMappingStore(String pathName, ObjectMapper objectMapper); + IdentifiableStore createInternToExternMappingStore( + String pathName, + ObjectMapper objectMapper); IdentifiableStore createSearchIndexStore(String pathName, ObjectMapper objectMapper); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ThreadPoolDefinition.java b/backend/src/main/java/com/bakdata/conquery/models/config/ThreadPoolDefinition.java index faba2ae254d..e6b6cfdb6fb 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ThreadPoolDefinition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ThreadPoolDefinition.java @@ -1,11 +1,11 @@ package com.bakdata.conquery.models.config; +import jakarta.validation.constraints.Min; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import com.google.common.util.concurrent.ThreadFactoryBuilder; import io.dropwizard.util.Duration; -import jakarta.validation.constraints.Min; import lombok.Getter; import lombok.Setter; import lombok.ToString; @@ -24,12 +24,12 @@ public class ThreadPoolDefinition { public ThreadPoolExecutor createService(String nameFormat) { final ThreadPoolExecutor executor = new ThreadPoolExecutor( - minThreads, - maxThreads, - keepAliveTime.getQuantity(), - keepAliveTime.getUnit(), - new LinkedBlockingQueue<>(), - new ThreadFactoryBuilder().setNameFormat(nameFormat).build() + minThreads, + maxThreads, + keepAliveTime.getQuantity(), + keepAliveTime.getUnit(), + new LinkedBlockingQueue<>(), + new ThreadFactoryBuilder().setNameFormat(nameFormat).build() ); executor.allowCoreThreadTimeOut(allowCoreThreadTimeOut); return executor; diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java index 04564133217..6cb08d82060 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java @@ -2,6 +2,10 @@ import static com.bakdata.conquery.io.storage.StoreMappings.*; +import jakarta.validation.Valid; +import jakarta.validation.Validator; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotNull; import java.io.File; import java.io.IOException; import java.nio.file.Path; @@ -18,11 +22,6 @@ import java.util.concurrent.TimeUnit; import java.util.function.Function; import javax.annotation.Nullable; -import jakarta.validation.Valid; -import jakarta.validation.Validator; -import jakarta.validation.constraints.Min; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.storage.IdentifiableStore; @@ -97,36 +96,35 @@ public class XodusStoreFactory implements StoreFactory { * creates two stores. Defining the expected stores like this, does not require a lot or complicated logic. */ public static final Set NAMESPACED_STORES = Set.of( - DATASET.storeInfo().getName(), - SECONDARY_IDS.storeInfo().getName(), - TABLES.storeInfo().getName(), - IMPORTS.storeInfo().getName(), - CONCEPTS.storeInfo().getName() + DATASET.storeInfo().getName(), + SECONDARY_IDS.storeInfo().getName(), + TABLES.storeInfo().getName(), + IMPORTS.storeInfo().getName(), + CONCEPTS.storeInfo().getName() ); public static final Set NAMESPACE_STORES = Sets.union( - NAMESPACED_STORES, - Set.of( - ID_MAPPING.storeInfo().getName() + BigStore.META, - ID_MAPPING.storeInfo().getName() + BigStore.DATA, - STRUCTURE.storeInfo().getName(), - WORKER_TO_BUCKETS.storeInfo().getName(), - ENTITY_PREVIEW.storeInfo().getName(), - ENTITY_TO_BUCKET.storeInfo().getName() - ) + NAMESPACED_STORES, + Set.of( + ID_MAPPING.storeInfo().getName() + BigStore.META, + ID_MAPPING.storeInfo().getName() + BigStore.DATA, + STRUCTURE.storeInfo().getName(), + WORKER_TO_BUCKETS.storeInfo().getName(), + ENTITY_PREVIEW.storeInfo().getName(), + ENTITY_TO_BUCKET.storeInfo().getName() + ) ); public static final Set WORKER_STORES = Sets.union( - NAMESPACED_STORES, - Set.of( - WORKER.storeInfo().getName(), - BUCKETS.storeInfo().getName(), - C_BLOCKS.storeInfo().getName() - ) + NAMESPACED_STORES, + Set.of( + WORKER.storeInfo().getName(), + BUCKETS.storeInfo().getName(), + C_BLOCKS.storeInfo().getName() + ) ); @JsonIgnore - private final transient Multimap - openStoresInEnv = - Multimaps.synchronizedSetMultimap(MultimapBuilder.hashKeys().hashSetValues().build()); + private final transient Multimap openStoresInEnv = Multimaps.synchronizedSetMultimap( + MultimapBuilder.hashKeys().hashSetValues().build()); private Path directory = Path.of("storage"); private boolean validateOnWrite = false; @NotNull @@ -211,7 +209,10 @@ public Collection discoverNamespaceStorages() { return loadNamespacedStores("dataset_", (storePath) -> new NamespaceStorage(this, storePath), NAMESPACE_STORES); } - private List loadNamespacedStores(String prefix, Function creator, Set storesToTest) { + private List loadNamespacedStores( + String prefix, + Function creator, + Set storesToTest) { final File baseDir = getDirectory().toFile(); if (baseDir.mkdirs()) { @@ -220,7 +221,8 @@ private List loadNamespacedStores(String pr final List storages = new ArrayList<>(); - for (File directory : Objects.requireNonNull(baseDir.listFiles((file, name) -> file.isDirectory() && name.startsWith(prefix)))) { + for (File directory : Objects.requireNonNull( + baseDir.listFiles((file, name) -> file.isDirectory() && name.startsWith(prefix)))) { final String name = directory.getName(); @@ -252,10 +254,11 @@ private boolean environmentHasStores(Environment env, Set storesToTest) final HashSet missing = Sets.newHashSet(storesToTest); allStoreNames.forEach(missing::remove); - log.warn("Environment did not contain all required stores. It is missing: {}. It had {}. {}", missing, allStoreNames, - loadEnvironmentWithMissingStores - ? "Loading environment anyway." - : "Skipping environment." + log.warn( + "Environment did not contain all required stores. It is missing: {}. It had {}. {}", + missing, + allStoreNames, + loadEnvironmentWithMissingStores ? "Loading environment anyway." : "Skipping environment." ); return loadEnvironmentWithMissingStores; @@ -269,48 +272,66 @@ public Collection discoverWorkerStorages() { @Override public SingletonStore createDatasetStore(String pathName, ObjectMapper objectMapper) { - return StoreMappings.singleton(createStore(findEnvironment(pathName), validator, DATASET, objectMapper, metricRegistry)); + return StoreMappings.singleton( + createStore(findEnvironment(pathName), validator, DATASET, objectMapper, metricRegistry)); } - public Store createStore(Environment environment, Validator validator, StoreMappings storeId, ObjectMapper objectMapper, - MetricRegistry metricRegistry) { + public Store createStore( + Environment environment, + Validator validator, + StoreMappings storeId, + ObjectMapper objectMapper, + MetricRegistry metricRegistry) { final StoreInfo storeInfo = storeId.storeInfo(); synchronized (openStoresInEnv) { - if (openStoresInEnv.get(environment).stream().map(XodusStore::getName).anyMatch(name -> storeInfo.getName().equals(name))) { + if (openStoresInEnv.get(environment) + .stream() + .map(XodusStore::getName) + .anyMatch( + name -> storeInfo.getName().equals(name))) { throw new IllegalStateException("Attempted to open an already opened store:" + storeInfo.getName()); } - final XodusStore xodusStoretore = new XodusStore(environment, storeInfo.getName(), this::closeStore, this::removeStore); + final XodusStore xodusStoretore = new XodusStore( + environment, + storeInfo.getName(), + this::closeStore, + this::removeStore); openStoresInEnv.put(environment, xodusStoretore); boolean cacheBinary = cacheBinaryStore.getOrDefault(storeId, Boolean.FALSE); CaffeineSpec binaryCacheSpec = CaffeineSpec.parse(""); - Store binaryStore = cacheBinary ? new CachedStore<>(xodusStoretore, binaryCacheSpec, metricRegistry) : xodusStoretore; + Store binaryStore = cacheBinary ? new CachedStore<>( + xodusStoretore, + binaryCacheSpec, + metricRegistry) : xodusStoretore; if (cacheBinary) { // Start caching of binary data in the background - Thread loadStoreThread = new Thread(binaryStore::loadData, "Cache binary store %s from %s".formatted(storeId, environment.getLocation())); + Thread loadStoreThread = new Thread( + binaryStore::loadData, + "Cache binary store %s from %s".formatted(storeId, environment.getLocation())); loadStoreThread.setDaemon(true); loadStoreThread.start(); } return new CachedStore<>( - new SerializingStore<>( - binaryStore, - validator, - objectMapper, - storeInfo.getKeyType(), - storeInfo.getValueType(), - isValidateOnWrite(), - isRemoveUnreadableFromStore(), - getUnreadableDataDumpDirectory(), - getReaderExecutorService() - ), - getCaffeineSpecParsed(), - metricRegistry + new SerializingStore<>( + binaryStore, + validator, + objectMapper, + storeInfo.getKeyType(), + storeInfo.getValueType(), + isValidateOnWrite(), + isRemoveUnreadableFromStore(), + getUnreadableDataDumpDirectory(), + getReaderExecutorService() + ), + getCaffeineSpecParsed(), + metricRegistry ); } } @@ -354,10 +375,12 @@ private void removeStore(XodusStore store) { public ExecutorService getReaderExecutorService() { if (readerExecutorService == null) { readerExecutorService = new ThreadPoolExecutor( - 1, getReaderWorkers(), - 5, TimeUnit.MINUTES, - new ArrayBlockingQueue<>(getReaderWorkers() * getBufferPerWorker()), - new ThreadPoolExecutor.CallerRunsPolicy() + 1, + getReaderWorkers(), + 5, + TimeUnit.MINUTES, + new ArrayBlockingQueue<>(getReaderWorkers() * getBufferPerWorker()), + new ThreadPoolExecutor.CallerRunsPolicy() ); } @@ -379,52 +402,61 @@ private void removeEnvironment(Environment env) { final List xodusStore = env.computeInReadonlyTransaction(env::getAllStoreNames); if (!xodusStore.isEmpty()) { - throw new IllegalStateException("Cannot delete environment, because it still contains these stores:" + xodusStore); + throw new IllegalStateException( + "Cannot delete environment, because it still contains these stores:" + xodusStore); } env.close(); try { FileUtil.deleteRecursive(Path.of(env.getLocation())); - } - catch (IOException e) { + } catch (IOException e) { log.error("Cannot delete directory of removed Environment[{}]", env.getLocation(), e); } } @Override - public IdentifiableStore createSecondaryIdDescriptionStore(String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, SECONDARY_IDS, objectMapper, metricRegistry)); + public IdentifiableStore createSecondaryIdDescriptionStore( + String pathName, + ObjectMapper objectMapper) { + return StoreMappings.identifiable( + createStore(findEnvironment(pathName), validator, SECONDARY_IDS, objectMapper, metricRegistry)); } @Override public IdentifiableStore
createTableStore(String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, TABLES, objectMapper, metricRegistry)); + return StoreMappings.identifiable( + createStore(findEnvironment(pathName), validator, TABLES, objectMapper, metricRegistry)); } @Override public IdentifiableStore> createConceptStore(String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, CONCEPTS, objectMapper, metricRegistry)); + return StoreMappings.identifiable( + createStore(findEnvironment(pathName), validator, CONCEPTS, objectMapper, metricRegistry)); } @Override public IdentifiableStore createImportStore(String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, IMPORTS, objectMapper, metricRegistry)); + return StoreMappings.identifiable( + createStore(findEnvironment(pathName), validator, IMPORTS, objectMapper, metricRegistry)); } @Override public IdentifiableStore createCBlockStore(String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, C_BLOCKS, objectMapper, metricRegistry)); + return StoreMappings.identifiable( + createStore(findEnvironment(pathName), validator, C_BLOCKS, objectMapper, metricRegistry)); } @Override public IdentifiableStore createBucketStore(String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, BUCKETS, objectMapper, metricRegistry)); + return StoreMappings.identifiable( + createStore(findEnvironment(pathName), validator, BUCKETS, objectMapper, metricRegistry)); } @Override public SingletonStore createWorkerInformationStore(String pathName, ObjectMapper objectMapper) { - return StoreMappings.singleton(createStore(findEnvironment(pathName), validator, WORKER, objectMapper, metricRegistry)); + return StoreMappings.singleton( + createStore(findEnvironment(pathName), validator, WORKER, objectMapper, metricRegistry)); } @Override @@ -432,8 +464,15 @@ public SingletonStore createIdMappingStore(String pathName, ObjectM final Environment environment = findEnvironment(pathName); synchronized (openStoresInEnv) { - final BigStore bigStore = - new BigStore<>(this, validator, environment, ID_MAPPING.storeInfo(), this::closeStore, this::removeStore, objectMapper, getReaderExecutorService()); + final BigStore bigStore = new BigStore<>( + this, + validator, + environment, + ID_MAPPING.storeInfo(), + this::closeStore, + this::removeStore, + objectMapper, + getReaderExecutorService()); openStoresInEnv.put(bigStore.getDataXodusStore().getEnvironment(), bigStore.getDataXodusStore()); openStoresInEnv.put(bigStore.getMetaXodusStore().getEnvironment(), bigStore.getMetaXodusStore()); @@ -443,17 +482,25 @@ public SingletonStore createIdMappingStore(String pathName, ObjectM @Override public SingletonStore createWorkerToBucketsStore(String pathName, ObjectMapper objectMapper) { - return StoreMappings.singleton(createStore(findEnvironment(pathName), validator, WORKER_TO_BUCKETS, objectMapper, metricRegistry)); + return StoreMappings.singleton( + createStore(findEnvironment(pathName), validator, WORKER_TO_BUCKETS, objectMapper, metricRegistry)); } @Override public SingletonStore createStructureStore(String pathName, ObjectMapper objectMapper) { - return StoreMappings.singleton(createStore(findEnvironment(pathName), validator, STRUCTURE, objectMapper, metricRegistry)); + return StoreMappings.singleton( + createStore(findEnvironment(pathName), validator, STRUCTURE, objectMapper, metricRegistry)); } @Override public IdentifiableStore createExecutionsStore(String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "executions")), validator, EXECUTIONS, objectMapper, metricRegistry)); + return StoreMappings.identifiable( + createStore( + findEnvironment(resolveSubDir(pathName, "executions")), + validator, + EXECUTIONS, + objectMapper, + metricRegistry)); } private Environment findEnvironment(File path) { @@ -472,37 +519,66 @@ private File resolveSubDir(String... subdirs) { @Override public IdentifiableStore createFormConfigStore(String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "formConfigs")), validator, FORM_CONFIG, objectMapper, metricRegistry)); + return StoreMappings.identifiable( + createStore( + findEnvironment(resolveSubDir(pathName, "formConfigs")), + validator, + FORM_CONFIG, + objectMapper, + metricRegistry)); } @Override public IdentifiableStore createUserStore(String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "users")), validator, AUTH_USER, objectMapper, metricRegistry)); + return StoreMappings.identifiable( + createStore( + findEnvironment(resolveSubDir(pathName, "users")), + validator, + AUTH_USER, + objectMapper, + metricRegistry)); } @Override public IdentifiableStore createRoleStore(String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "roles")), validator, AUTH_ROLE, objectMapper, metricRegistry)); + return StoreMappings.identifiable( + createStore( + findEnvironment(resolveSubDir(pathName, "roles")), + validator, + AUTH_ROLE, + objectMapper, + metricRegistry)); } @Override public IdentifiableStore createGroupStore(String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "groups")), validator, AUTH_GROUP, objectMapper, metricRegistry)); + return StoreMappings.identifiable( + createStore( + findEnvironment(resolveSubDir(pathName, "groups")), + validator, + AUTH_GROUP, + objectMapper, + metricRegistry)); } @Override - public IdentifiableStore createInternToExternMappingStore(String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, INTERN_TO_EXTERN, objectMapper, metricRegistry)); + public IdentifiableStore createInternToExternMappingStore( + String pathName, + ObjectMapper objectMapper) { + return StoreMappings.identifiable( + createStore(findEnvironment(pathName), validator, INTERN_TO_EXTERN, objectMapper, metricRegistry)); } @Override public IdentifiableStore createSearchIndexStore(String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, SEARCH_INDEX, objectMapper, metricRegistry)); + return StoreMappings.identifiable( + createStore(findEnvironment(pathName), validator, SEARCH_INDEX, objectMapper, metricRegistry)); } @Override public SingletonStore createPreviewStore(String pathName, ObjectMapper objectMapper) { - return StoreMappings.singleton(createStore(findEnvironment(pathName), validator, ENTITY_PREVIEW, objectMapper, metricRegistry)); + return StoreMappings.singleton( + createStore(findEnvironment(pathName), validator, ENTITY_PREVIEW, objectMapper, metricRegistry)); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/auth/ApiKeyClientFilterProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/auth/ApiKeyClientFilterProvider.java index ccec1d9d192..d39421323d6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/auth/ApiKeyClientFilterProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/auth/ApiKeyClientFilterProvider.java @@ -1,11 +1,12 @@ package com.bakdata.conquery.models.config.auth; -import com.bakdata.conquery.io.cps.CPSType; -import com.fasterxml.jackson.annotation.JsonIgnore; import jakarta.validation.constraints.NotEmpty; import jakarta.ws.rs.client.ClientRequestFilter; +import com.bakdata.conquery.io.cps.CPSType; +import com.fasterxml.jackson.annotation.JsonIgnore; + @CPSType(id = "API_KEY", base = AuthenticationClientFilterProvider.class) public record ApiKeyClientFilterProvider(@NotEmpty String apiKey) implements AuthenticationClientFilterProvider { diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/auth/AuthenticationClientFilterProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/auth/AuthenticationClientFilterProvider.java index 5fe9519e803..78396562363 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/auth/AuthenticationClientFilterProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/auth/AuthenticationClientFilterProvider.java @@ -1,8 +1,9 @@ package com.bakdata.conquery.models.config.auth; +import jakarta.ws.rs.client.ClientRequestFilter; + import com.bakdata.conquery.io.cps.CPSBase; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import jakarta.ws.rs.client.ClientRequestFilter; @CPSBase @JsonTypeInfo(property = "type", use = JsonTypeInfo.Id.CUSTOM) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/auth/AuthenticationConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/auth/AuthenticationConfig.java index 328cf3a969f..28b4210a089 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/auth/AuthenticationConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/auth/AuthenticationConfig.java @@ -1,10 +1,11 @@ package com.bakdata.conquery.models.config.auth; -import com.bakdata.conquery.models.auth.web.AuthCookieFilter; -import io.dropwizard.util.Duration; import jakarta.validation.constraints.NotNull; import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.NewCookie; + +import com.bakdata.conquery.models.auth.web.AuthCookieFilter; +import io.dropwizard.util.Duration; import lombok.Getter; import lombok.NoArgsConstructor; @@ -19,28 +20,28 @@ public class AuthenticationConfig { public NewCookie createAuthCookie(ContainerRequestContext request, String token) { return new NewCookie( - AuthCookieFilter.ACCESS_TOKEN, - token, - "/", - null, - 0, - null, - Long.valueOf(adminEndCookieDuration.toSeconds()).intValue(), - null, - request.getSecurityContext().isSecure(), - true + AuthCookieFilter.ACCESS_TOKEN, + token, + "/", + null, + 0, + null, + Long.valueOf(adminEndCookieDuration.toSeconds()).intValue(), + null, + request.getSecurityContext().isSecure(), + true ); } public static NewCookie expireCookie(String cookieName) { return new NewCookie( - cookieName, - null, - "/", - null, - null, - 0, - false + cookieName, + null, + "/", + null, + null, + 0, + false ); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/auth/AuthenticationRealmFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/auth/AuthenticationRealmFactory.java index dbf2ba96335..ea7a8c39bff 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/auth/AuthenticationRealmFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/auth/AuthenticationRealmFactory.java @@ -11,7 +11,7 @@ @JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "type") @CPSBase public interface AuthenticationRealmFactory { - + /** * Gets the realm specified in the configuration. * @@ -21,5 +21,8 @@ public interface AuthenticationRealmFactory { * @return The realm. */ @JsonIgnore - ConqueryAuthenticationRealm createRealm(Environment environment, ConqueryConfig config, AuthorizationController authorizationController); + ConqueryAuthenticationRealm createRealm( + Environment environment, + ConqueryConfig config, + AuthorizationController authorizationController); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/auth/AuthorizationConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/auth/AuthorizationConfig.java index a6dbd3fdf15..5051dcb9f8f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/auth/AuthorizationConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/auth/AuthorizationConfig.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.config.auth; +import jakarta.validation.constraints.NotNull; import java.util.List; import com.bakdata.conquery.apiv1.auth.ProtoRole; @@ -8,7 +9,6 @@ import com.bakdata.conquery.models.auth.UserManageable; import com.bakdata.conquery.models.auth.permissions.ExecutionPermission; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import jakarta.validation.constraints.NotNull; /** * Configurations of this type define the initial users with their permissions @@ -25,6 +25,7 @@ public interface AuthorizationConfig { */ @NotNull List getInitialUsers(); + List getInitialRoles(); /** diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/auth/DefaultAuthorizationConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/auth/DefaultAuthorizationConfig.java index 0ba2a1f2846..f2e3b0eef68 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/auth/DefaultAuthorizationConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/auth/DefaultAuthorizationConfig.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.config.auth; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; import java.util.Collections; import java.util.List; import java.util.Set; @@ -11,8 +13,6 @@ import com.bakdata.conquery.models.auth.permissions.DatasetPermission; import com.bakdata.conquery.models.auth.permissions.SuperPermission; import io.dropwizard.validation.ValidationMethod; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotEmpty; import lombok.Data; @CPSType(base = AuthorizationConfig.class, id = "DEFAULT") @@ -20,16 +20,17 @@ public class DefaultAuthorizationConfig implements AuthorizationConfig { @Valid - private List initialRoles = List.of(ProtoRole.builder() - .name("admin") - .permissions(Set.of(AdminPermission.DOMAIN)) - .build()); + private List initialRoles = List.of( + ProtoRole.builder().name("admin").permissions(Set.of(AdminPermission.DOMAIN)).build()); @Valid private List initialUsers = Collections.emptyList(); @NotEmpty - private List overviewScope = List.of(DatasetPermission.DOMAIN, AdminPermission.DOMAIN, SuperPermission.DOMAIN); + private List overviewScope = List.of( + DatasetPermission.DOMAIN, + AdminPermission.DOMAIN, + SuperPermission.DOMAIN); @ValidationMethod(message = "No initial entities defined. Access will not be possible") public boolean isInitialAccessPossible() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/auth/DevelopmentAuthorizationConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/auth/DevelopmentAuthorizationConfig.java index 278b491c5f2..8a7007e38ff 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/auth/DevelopmentAuthorizationConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/auth/DevelopmentAuthorizationConfig.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.config.auth; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; import java.util.List; import java.util.Set; @@ -9,8 +11,6 @@ import com.bakdata.conquery.models.auth.permissions.AdminPermission; import com.bakdata.conquery.models.auth.permissions.DatasetPermission; import com.bakdata.conquery.models.auth.permissions.SuperPermission; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; import lombok.Getter; @@ -18,20 +18,23 @@ @Getter public class DevelopmentAuthorizationConfig implements AuthorizationConfig { - private List initialRoles = List.of(ProtoRole.builder() - .name("admin") - .permissions(Set.of(AdminPermission.DOMAIN)) - .build()); + private List initialRoles = List.of( + ProtoRole.builder().name("admin").permissions(Set.of(AdminPermission.DOMAIN)).build()); @NotEmpty - private List initialUsers = List.of(ProtoUser.builder() - .name("SUPERUSER") - .label("SUPERUSER") - .permissions(Set.of("*")) - .roles(Set.of("admin")) - .build()); + private List initialUsers = List.of( + ProtoUser.builder() + .name("SUPERUSER") + .label("SUPERUSER") + .permissions(Set.of("*")) + .roles( + Set.of("admin")) + .build()); @NotNull - private List overviewScope = List.of(DatasetPermission.DOMAIN, AdminPermission.DOMAIN, SuperPermission.DOMAIN); + private List overviewScope = List.of( + DatasetPermission.DOMAIN, + AdminPermission.DOMAIN, + SuperPermission.DOMAIN); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/auth/IntrospectionDelegatingRealmFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/auth/IntrospectionDelegatingRealmFactory.java index 3501c01c672..d09545227e5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/auth/IntrospectionDelegatingRealmFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/auth/IntrospectionDelegatingRealmFactory.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.config.auth; +import jakarta.ws.rs.client.Client; import java.io.PrintWriter; import java.util.List; import java.util.Map; @@ -19,7 +20,6 @@ import io.dropwizard.core.setup.Environment; import io.dropwizard.servlets.tasks.Task; import io.dropwizard.validation.ValidationMethod; -import jakarta.ws.rs.client.Client; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -46,10 +46,14 @@ public class IntrospectionDelegatingRealmFactory extends Configuration { private transient AuthzClient authClient; - public ConqueryAuthenticationRealm createRealm(Environment environment, AuthorizationController authorizationController) { + public ConqueryAuthenticationRealm createRealm( + Environment environment, + AuthorizationController authorizationController) { // Register token extractor for JWT Tokens - AuthFilter.registerTokenExtractor(JWTokenHandler.JWTokenExtractor.class, environment.jersey().getResourceConfig()); + AuthFilter.registerTokenExtractor( + JWTokenHandler.JWTokenExtractor.class, + environment.jersey().getResourceConfig()); // At start up, try tp retrieve the idp client api object if possible. If the idp service is not up don't fail start up. authClient = getAuthClient(false); @@ -79,7 +83,7 @@ public void execute(Map> parameters, PrintWriter output) th * @return */ @JsonIgnore - public String getTokenEndpoint(){ + public String getTokenEndpoint() { return getAuthClient(true).getServerConfiguration().getTokenEndpoint(); } @@ -106,15 +110,15 @@ public final ClientAuthentication getClientAuthentication() { @JsonIgnore public AuthzClient getAuthClient(boolean exceptionOnFailedRetrieval) { - if(authClient != null) { + if (authClient != null) { return authClient; } try { // This tries to contact the identity providers discovery endpoint and can possibly timeout return AuthzClient.create(this); } catch (RuntimeException e) { - log.warn("Unable to establish connection to auth server.", log.isTraceEnabled()? e : null ); - if(exceptionOnFailedRetrieval) { + log.warn("Unable to establish connection to auth server.", log.isTraceEnabled() ? e : null); + if (exceptionOnFailedRetrieval) { throw e; } } @@ -138,19 +142,19 @@ public boolean isResourceFilled() { @JsonIgnore @ValidationMethod(message = "Secret not found") public boolean isSecretFilled() { - if(credentials == null) { + if (credentials == null) { return false; } Object secret = credentials.get(CONFIDENTIAL_CREDENTIAL); - if(secret == null) { + if (secret == null) { return false; } - if(!(secret instanceof String)) { + if (!(secret instanceof String)) { return false; } - return !((String)secret).isBlank(); + return !((String) secret).isBlank(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/auth/JwtPkceVerifyingRealmFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/auth/JwtPkceVerifyingRealmFactory.java index 11ab2d08a41..76bb18b35ca 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/auth/JwtPkceVerifyingRealmFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/auth/JwtPkceVerifyingRealmFactory.java @@ -1,5 +1,16 @@ package com.bakdata.conquery.models.config.auth; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.core.Cookie; +import jakarta.ws.rs.core.HttpHeaders; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.NewCookie; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.UriBuilder; import java.io.IOException; import java.net.URI; import java.security.PublicKey; @@ -13,17 +24,6 @@ import java.util.function.BiFunction; import java.util.function.Supplier; import java.util.stream.Collectors; -import jakarta.validation.constraints.Min; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; -import jakarta.ws.rs.client.Client; -import jakarta.ws.rs.container.ContainerRequestContext; -import jakarta.ws.rs.core.Cookie; -import jakarta.ws.rs.core.HttpHeaders; -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.NewCookie; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.apiv1.RequestHelper; import com.bakdata.conquery.io.cps.CPSType; @@ -147,14 +147,17 @@ public boolean isConfigurationAvailable() { * It can be retrieved from the IDP. */ public record IdpConfiguration( - @NonNull Map signingKeys, - @NonNull URI authorizationEndpoint, - @NonNull URI tokenEndpoint, - @NonNull URI logoutEndpoint, - @NotEmpty String issuer) { + @NonNull Map signingKeys, + @NonNull URI authorizationEndpoint, + @NonNull URI tokenEndpoint, + @NonNull URI logoutEndpoint, + @NotEmpty String issuer) { } - public ConqueryAuthenticationRealm createRealm(Environment environment, ConqueryConfig config, AuthorizationController authorizationController) { + public ConqueryAuthenticationRealm createRealm( + Environment environment, + ConqueryConfig config, + AuthorizationController authorizationController) { final List> additionalVerifiers = new ArrayList<>(); for (String additionalTokenCheck : additionalTokenChecks) { @@ -168,16 +171,23 @@ public ConqueryAuthenticationRealm createRealm(Environment environment, Conquery // Add login schema for admin UI final DropwizardResourceConfig jerseyAdminUi = authorizationController.getAdminServlet().getJerseyConfigUI(); RedirectingAuthFilter.registerLoginInitiator(jerseyAdminUi, this::initiateLogin, "jwt-initiator"); - RedirectingAuthFilter.registerAuthAttemptChecker(jerseyAdminUi, this::checkAndRedeemAuthzCode, "jwt-authz-redeemer"); - RedirectingAuthFilter.registerAuthAttemptChecker(jerseyAdminUi, this::checkAndRedeemRefreshToken, "jwt-refresh-redeemer"); - - return new JwtPkceVerifyingRealm(idpConfigurationSupplier, - client, - additionalVerifiers, - alternativeIdClaims, - authorizationController.getStorage(), - tokenLeeway, - environment.getValidator() + RedirectingAuthFilter.registerAuthAttemptChecker( + jerseyAdminUi, + this::checkAndRedeemAuthzCode, + "jwt-authz-redeemer"); + RedirectingAuthFilter.registerAuthAttemptChecker( + jerseyAdminUi, + this::checkAndRedeemRefreshToken, + "jwt-refresh-redeemer"); + + return new JwtPkceVerifyingRealm( + idpConfigurationSupplier, + client, + additionalVerifiers, + alternativeIdClaims, + authorizationController.getStorage(), + tokenLeeway, + environment.getValidator() ); } @@ -194,13 +204,12 @@ private Supplier> getIdpOptionsSupplier(Environment e // check again since we are now in an exclusive section if (idpConfiguration == null) { - final Client httpClient = new JerseyClientBuilder(environment).using(config.getJerseyClient()) - .build(this.getClass().getSimpleName()); + final Client httpClient = new JerseyClientBuilder(environment).using( + config.getJerseyClient()).build(this.getClass().getSimpleName()); try { // retrieve the configuration and cache it idpConfiguration = retrieveIdpConfiguration(httpClient); - } - finally { + } finally { httpClient.close(); } } @@ -224,11 +233,8 @@ public IdpConfiguration retrieveIdpConfiguration(final Client client) { JsonNode response; try { - response = client.target(wellKnownEndpoint) - .request(MediaType.APPLICATION_JSON_TYPE) - .get(JsonNode.class); - } - catch (Exception e) { + response = client.target(wellKnownEndpoint).request(MediaType.APPLICATION_JSON_TYPE).get(JsonNode.class); + } catch (Exception e) { log.warn("Unable to retrieve configuration from {}", wellKnownEndpoint, e); return null; } @@ -246,12 +252,9 @@ public IdpConfiguration retrieveIdpConfiguration(final Client client) { JWKs jwks; try { - jwks = client.target(jwksUri) - .request(MediaType.APPLICATION_JSON_TYPE) - .get(JWKs.class); + jwks = client.target(jwksUri).request(MediaType.APPLICATION_JSON_TYPE).get(JWKs.class); - } - catch (Exception e) { + } catch (Exception e) { log.warn("Unable to retrieve jwks from {}", wellKnownEndpoint, e); return null; } @@ -262,15 +265,21 @@ public IdpConfiguration retrieveIdpConfiguration(final Client client) { // Filter for keys that are used for signing (discard encryption keys) - final Map signingKeys = jwks.getKeys().stream() - .filter(jwk -> JWK.Use.SIG.asString().equals(jwk.getPublicKeyUse())) - .collect(Collectors.toMap(JWK::getKeyId, JwtPkceVerifyingRealmFactory::getPublicKey)); + final Map signingKeys = jwks.getKeys() + .stream() + .filter( + jwk -> JWK.Use.SIG.asString().equals(jwk.getPublicKeyUse())) + .collect( + Collectors.toMap(JWK::getKeyId, JwtPkceVerifyingRealmFactory::getPublicKey)); if (signingKeys.isEmpty()) { - throw new IllegalStateException("No signing keys could be retrieved from IDP. Received these JWKs (Key Ids):" + jwks.getKeys() - .stream() - .map(JWK::getKeyId).toList()); + throw new IllegalStateException( + "No signing keys could be retrieved from IDP. Received these JWKs (Key Ids):" + jwks.getKeys() + .stream() + .map( + JWK::getKeyId) + .toList()); } return new IdpConfiguration(signingKeys, authorizationEndpoint, tokenEndpoint, logoutEndpoint, issuer); @@ -327,11 +336,21 @@ private URI initiateLogin(ContainerRequestContext request) { } JwtPkceVerifyingRealmFactory.IdpConfiguration idpConfiguration = idpConfigurationOpt.get(); return UriBuilder.fromUri(idpConfiguration.authorizationEndpoint()) - .queryParam("response_type", "code") - .queryParam("client_id", client) - .queryParam("redirect_uri", UriBuilder.fromUri(RequestHelper.getRequestURL(request)).path(AdminServlet.ADMIN_UI).build()) - .queryParam("scope", "openid") - .queryParam("state", UUID.randomUUID()).build(); + .queryParam( + "response_type", + "code") + .queryParam("client_id", client) + .queryParam( + "redirect_uri", + UriBuilder.fromUri(RequestHelper.getRequestURL(request)) + .path( + AdminServlet.ADMIN_UI) + .build()) + .queryParam("scope", "openid") + .queryParam( + "state", + UUID.randomUUID()) + .build(); } @@ -347,14 +366,17 @@ private Response checkAndRedeemAuthzCode(ContainerRequestContext request) { } // Build the original redirect uri (the request uri without the query added by the IDP) - final URI redirectedUri = - UriBuilder.fromUri(RequestHelper.getRequestURL(request)).replacePath(request.getUriInfo().getAbsolutePath().getPath()).replaceQuery("").build(); + final URI redirectedUri = UriBuilder.fromUri(RequestHelper.getRequestURL(request)) + .replacePath( + request.getUriInfo().getAbsolutePath().getPath()) + .replaceQuery("") + .build(); log.trace("Redirect URI: {}", redirectedUri); // Prepare code for exchange with access token final AuthorizationCodeGrant authzGrant = new AuthorizationCodeGrant( - new AuthorizationCode(code), - redirectedUri + new AuthorizationCode(code), + redirectedUri ); // Redeem code @@ -385,7 +407,8 @@ private Response checkAndRedeemRefreshToken(ContainerRequestContext request) { } // Redeem refresh token for a new access token (+ new refresh token) - final AccessTokenResponse tokenResponse = getTokenResponse(new RefreshTokenGrant(new RefreshToken(refreshToken.getValue()))); + final AccessTokenResponse tokenResponse = getTokenResponse( + new RefreshTokenGrant(new RefreshToken(refreshToken.getValue()))); if (tokenResponse == null) { return null; } @@ -394,18 +417,22 @@ private Response checkAndRedeemRefreshToken(ContainerRequestContext request) { final Cookie accessTokenCookie = prepareAccessTokenCookie(request, tokenResponse); final NewCookie refreshTokenCookie = prepareRefreshTokenCookie(request, tokenResponse); - return prepareRedirectResponse(request.getUriInfo().getRequestUriBuilder().replaceQuery("").build(), accessTokenCookie, refreshTokenCookie); + return prepareRedirectResponse( + request.getUriInfo().getRequestUriBuilder().replaceQuery("").build(), + accessTokenCookie, + refreshTokenCookie); } /** * Prepares a redirect response which also saves the access and refresh token on the client. */ private Response prepareRedirectResponse(URI uri, Cookie accessTokenCookie, NewCookie refreshTokenCookie) { - return Response - .seeOther(uri) - .header(HttpHeaders.SET_COOKIE, accessTokenCookie) - .header(HttpHeaders.SET_COOKIE, refreshTokenCookie) - .build(); + return Response.seeOther(uri) + .header(HttpHeaders.SET_COOKIE, accessTokenCookie) + .header( + HttpHeaders.SET_COOKIE, + refreshTokenCookie) + .build(); } private Cookie prepareAccessTokenCookie(ContainerRequestContext request, AccessTokenResponse tokenResponse) { @@ -427,16 +454,16 @@ private NewCookie prepareRefreshTokenCookie(ContainerRequestContext request, Acc return new NewCookie( - REFRESH_TOKEN, - refreshToken.getValue(), - "/", - null, - 0, - null, - 900, //The Constructor requires us to set a maxAge even though it should be optional. 30 minutes should be okay - exp, - request.getSecurityContext().isSecure(), - true + REFRESH_TOKEN, + refreshToken.getValue(), + "/", + null, + 0, + null, + 900, //The Constructor requires us to set a maxAge even though it should be optional. 30 minutes should be okay + exp, + request.getSecurityContext().isSecure(), + true ); } @@ -457,9 +484,9 @@ private AccessTokenResponse getTokenResponse(AuthorizationGrant authzGrant) { // Send the auth code/refresh token to the IDP to redeem them for a new access and refresh token final TokenRequest tokenRequest = new TokenRequest( - UriBuilder.fromUri(idpConfiguration.tokenEndpoint()).build(), - new ClientID(client), - authzGrant + UriBuilder.fromUri(idpConfiguration.tokenEndpoint()).build(), + new ClientID(client), + authzGrant ); // Get the response @@ -469,10 +496,12 @@ private AccessTokenResponse getTokenResponse(AuthorizationGrant authzGrant) { // Check if the response was valid if (!response.indicatesSuccess()) { HTTPResponse httpResponse = response.toHTTPResponse(); - log.warn("Unable to retrieve access token from auth server. Request: {} Response: {}", httpRequest.getURL(), httpResponse.getBody()); + log.warn( + "Unable to retrieve access token from auth server. Request: {} Response: {}", + httpRequest.getURL(), + httpResponse.getBody()); return null; - } - else if (!(response instanceof AccessTokenResponse)) { + } else if (!(response instanceof AccessTokenResponse)) { log.warn("Unknown token response {}.", response.getClass().getName()); return null; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/auth/LocalAuthenticationConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/auth/LocalAuthenticationConfig.java index fb9adc491c4..ffb5d69769a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/auth/LocalAuthenticationConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/auth/LocalAuthenticationConfig.java @@ -1,13 +1,13 @@ package com.bakdata.conquery.models.config.auth; -import java.io.File; -import java.net.URI; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.UriBuilder; +import java.io.File; +import java.net.URI; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; import com.bakdata.conquery.apiv1.RequestHelper; import com.bakdata.conquery.io.cps.CPSType; @@ -54,14 +54,14 @@ public class LocalAuthenticationConfig implements AuthenticationRealmFactory { @MinDuration(value = 1, unit = TimeUnit.MINUTES) private Duration jwtDuration = Duration.hours(12); - + /** * The name of the folder the store lives in. */ @NotEmpty private String storeName = "authenticationStore"; - + @NotNull private File directory = new File("storage"); @@ -81,33 +81,45 @@ boolean isStorageEncrypted() { // and system property return System.getProperty("exodus.cipherId") != null; } - + @Override - public ConqueryAuthenticationRealm createRealm(Environment environment, ConqueryConfig config, AuthorizationController authorizationController) { + public ConqueryAuthenticationRealm createRealm( + Environment environment, + ConqueryConfig config, + AuthorizationController authorizationController) { // Token extractor is not needed because this realm depends on the ConqueryTokenRealm - AuthFilter.registerTokenExtractor(JWTokenHandler.JWTokenExtractor.class, environment.jersey().getResourceConfig()); + AuthFilter.registerTokenExtractor( + JWTokenHandler.JWTokenExtractor.class, + environment.jersey().getResourceConfig()); - log.info("Performing benchmark for default hash function (bcrypt) with max_milliseconds={}", BCRYPT_MAX_MILLISECONDS); + log.info( + "Performing benchmark for default hash function (bcrypt) with max_milliseconds={}", + BCRYPT_MAX_MILLISECONDS); final BenchmarkResult result = SystemChecker.benchmarkBcrypt(BCRYPT_MAX_MILLISECONDS); final BcryptFunction prototype = result.getPrototype(); - log.info("Using bcrypt with {} logarithmic rounds. Elapsed time={}", prototype.getLogarithmicRounds(), result.getElapsed()); + log.info( + "Using bcrypt with {} logarithmic rounds. Elapsed time={}", + prototype.getLogarithmicRounds(), + result.getElapsed()); LocalAuthenticationRealm realm = new LocalAuthenticationRealm( - environment.getValidator(), - Jackson.copyMapperAndInjectables(Jackson.BINARY_MAPPER), - authorizationController.getConqueryTokenRealm(), - storeName, - directory, - passwordStoreConfig, - jwtDuration, - prototype, - CaffeineSpec.parse(caffeineSpec), - environment.metrics() + environment.getValidator(), + Jackson.copyMapperAndInjectables(Jackson.BINARY_MAPPER), + authorizationController.getConqueryTokenRealm(), + storeName, + directory, + passwordStoreConfig, + jwtDuration, + prototype, + CaffeineSpec.parse(caffeineSpec), + environment.metrics() ); - UserAuthenticationManagementProcessor processor = new UserAuthenticationManagementProcessor(realm, authorizationController.getStorage()); + UserAuthenticationManagementProcessor processor = new UserAuthenticationManagementProcessor( + realm, + authorizationController.getStorage()); // Register resources for users to exchange username and password for an access token registerAdminUnprotectedAuthenticationResources(authorizationController.getUnprotectedAuthAdmin(), realm); @@ -121,30 +133,36 @@ public ConqueryAuthenticationRealm createRealm(Environment environment, Conquery return realm; } - //////////////////// RESOURCE REGISTRATION //////////////////// - public void registerAdminUnprotectedAuthenticationResources(DropwizardResourceConfig jerseyConfig, LocalAuthenticationRealm realm) { + //////////////////// RESOURCE REGISTRATION + public void registerAdminUnprotectedAuthenticationResources( + DropwizardResourceConfig jerseyConfig, + LocalAuthenticationRealm realm) { jerseyConfig.register(new TokenResource(realm)); jerseyConfig.register(LoginResource.class); } - public void registerApiUnprotectedAuthenticationResources(DropwizardResourceConfig jerseyConfig, LocalAuthenticationRealm realm) { + public void registerApiUnprotectedAuthenticationResources( + DropwizardResourceConfig jerseyConfig, + LocalAuthenticationRealm realm) { jerseyConfig.register(new TokenResource(realm)); } - public void registerAuthenticationAdminResources(DropwizardResourceConfig jerseyConfig, UserAuthenticationManagementProcessor userProcessor) { + public void registerAuthenticationAdminResources( + DropwizardResourceConfig jerseyConfig, + UserAuthenticationManagementProcessor userProcessor) { jerseyConfig.register(userProcessor); jerseyConfig.register(UserAuthenticationManagementResource.class); } - private Function loginProvider(DropwizardResourceConfig unprotectedAuthAdmin) { + private Function loginProvider(DropwizardResourceConfig unprotectedAuthAdmin) { return (ContainerRequestContext request) -> { return UriBuilder.fromPath(unprotectedAuthAdmin.getUrlPattern()) - .path(LoginResource.class) - .queryParam(RedirectingAuthFilter.REDIRECT_URI, UriBuilder.fromUri(RequestHelper.getRequestURL(request)) - .path(AdminServlet.ADMIN_UI) - .build()) - .build(); + .path(LoginResource.class) + .queryParam( + RedirectingAuthFilter.REDIRECT_URI, + UriBuilder.fromUri(RequestHelper.getRequestURL(request)).path(AdminServlet.ADMIN_UI).build()) + .build(); }; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/auth/OIDCAuthorizationCodeFlowRealmFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/auth/OIDCAuthorizationCodeFlowRealmFactory.java index fafad825d30..79d11f5c67f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/auth/OIDCAuthorizationCodeFlowRealmFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/auth/OIDCAuthorizationCodeFlowRealmFactory.java @@ -21,7 +21,10 @@ public class OIDCAuthorizationCodeFlowRealmFactory implements AuthenticationReal private IntrospectionDelegatingRealmFactory client; @Override - public ConqueryAuthenticationRealm createRealm(Environment environment, ConqueryConfig config, AuthorizationController authorizationController) { + public ConqueryAuthenticationRealm createRealm( + Environment environment, + ConqueryConfig config, + AuthorizationController authorizationController) { return client.createRealm(environment, authorizationController); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/auth/OIDCResourceOwnerPasswordCredentialRealmFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/auth/OIDCResourceOwnerPasswordCredentialRealmFactory.java index 8726be6e4f3..594386b3a8a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/auth/OIDCResourceOwnerPasswordCredentialRealmFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/auth/OIDCResourceOwnerPasswordCredentialRealmFactory.java @@ -7,8 +7,8 @@ import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.resources.unprotected.LoginResource; import com.bakdata.conquery.resources.unprotected.TokenResource; -import io.dropwizard.jersey.DropwizardResourceConfig; import io.dropwizard.core.setup.Environment; +import io.dropwizard.jersey.DropwizardResourceConfig; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -25,26 +25,36 @@ public class OIDCResourceOwnerPasswordCredentialRealmFactory implements Authenti private IntrospectionDelegatingRealmFactory client; - @Override - public ConqueryAuthenticationRealm createRealm(Environment environment, ConqueryConfig config, AuthorizationController authorizationController) { + public ConqueryAuthenticationRealm createRealm( + Environment environment, + ConqueryConfig config, + AuthorizationController authorizationController) { // Register processor that does the actual exchange of user credentials for an access token IdpDelegatingAccessTokenCreator idpDelegatingAccessTokenCreator = new IdpDelegatingAccessTokenCreator(client); // Register resources for users to exchange username and password for an access token - registerAdminUnprotectedAuthenticationResources(authorizationController.getUnprotectedAuthAdmin(), idpDelegatingAccessTokenCreator); - registerApiUnprotectedAuthenticationResources(authorizationController.getUnprotectedAuthApi(), idpDelegatingAccessTokenCreator); + registerAdminUnprotectedAuthenticationResources( + authorizationController.getUnprotectedAuthAdmin(), + idpDelegatingAccessTokenCreator); + registerApiUnprotectedAuthenticationResources( + authorizationController.getUnprotectedAuthApi(), + idpDelegatingAccessTokenCreator); return client.createRealm(environment, authorizationController); } - public void registerAdminUnprotectedAuthenticationResources(DropwizardResourceConfig jerseyConfig, IdpDelegatingAccessTokenCreator idpDelegatingAccessTokenCreator) { + public void registerAdminUnprotectedAuthenticationResources( + DropwizardResourceConfig jerseyConfig, + IdpDelegatingAccessTokenCreator idpDelegatingAccessTokenCreator) { jerseyConfig.register(new TokenResource(idpDelegatingAccessTokenCreator)); jerseyConfig.register(LoginResource.class); } - public void registerApiUnprotectedAuthenticationResources(DropwizardResourceConfig jerseyConfig, IdpDelegatingAccessTokenCreator idpDelegatingAccessTokenCreator) { + public void registerApiUnprotectedAuthenticationResources( + DropwizardResourceConfig jerseyConfig, + IdpDelegatingAccessTokenCreator idpDelegatingAccessTokenCreator) { jerseyConfig.register(new TokenResource(idpDelegatingAccessTokenCreator)); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/search/InternalSearchConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/search/InternalSearchConfig.java index a940fd08046..f137b3353d9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/search/InternalSearchConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/search/InternalSearchConfig.java @@ -1,12 +1,12 @@ package com.bakdata.conquery.models.config.search; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotNull; import java.net.URI; import java.util.List; import java.util.stream.Collectors; import javax.annotation.Nullable; -import jakarta.validation.constraints.Min; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.apiv1.FilterTemplate; import com.bakdata.conquery.apiv1.LabelMap; @@ -53,25 +53,34 @@ public SearchProcessor createSearchProcessor(Environment environment, DatasetId @VisibleForTesting public TrieSearch createSearch(Searchable searchable) { - return switch(searchable) { + return switch (searchable) { case FilterTemplate temp -> getFilterTemplateSearch(temp); case LabelMap labelMap -> getLabelMapSearch(labelMap); - default -> new TrieSearch<>(searchable.isGenerateSuffixes() ? getNgramLength() : Integer.MAX_VALUE, getSearchSplitChars()); + default -> new TrieSearch<>( + searchable.isGenerateSuffixes() ? getNgramLength() : Integer.MAX_VALUE, + getSearchSplitChars()); }; } private TrieSearch getLabelMapSearch(LabelMap labelMap) { - TrieSearch search = new TrieSearch<>(labelMap.isGenerateSuffixes() ? getNgramLength() : Integer.MAX_VALUE, getSearchSplitChars()); + TrieSearch search = new TrieSearch<>( + labelMap.isGenerateSuffixes() ? getNgramLength() : Integer.MAX_VALUE, + getSearchSplitChars()); BiMap delegate = labelMap.getDelegate(); FilterId id = labelMap.getId(); - final List collected = delegate.entrySet().stream() - .map(entry -> new FrontendValue(entry.getKey(), entry.getValue())) - .toList(); + final List collected = delegate.entrySet() + .stream() + .map( + entry -> new FrontendValue(entry.getKey(), entry.getValue())) + .toList(); if (log.isTraceEnabled()) { - log.trace("Labels for {}: `{}`", id, collected.stream().map(FrontendValue::toString).collect(Collectors.toList())); + log.trace( + "Labels for {}: `{}`", + id, + collected.stream().map(FrontendValue::toString).collect(Collectors.toList())); } StopWatch timer = StopWatch.createStarted(); @@ -97,18 +106,19 @@ private TrieSearch getFilterTemplateSearch(FilterTemplate temp) { final FrontendValueIndex search; try { - search = temp.getIndexService().getIndex(new FrontendValueIndexKey( - resolvedURI, - temp.getColumnValue(), - temp.getValue(), - temp.getOptionValue(), - () -> new TrieSearch<>( + search = temp.getIndexService() + .getIndex( + new FrontendValueIndexKey( + resolvedURI, + temp.getColumnValue(), + temp.getValue(), + temp.getOptionValue(), + () -> new TrieSearch<>( temp.isGenerateSuffixes() ? temp.getMinSuffixLength() : Integer.MAX_VALUE, getSearchSplitChars() - ) - )); - } - catch (IndexCreationException e) { + ) + )); + } catch (IndexCreationException e) { throw new RuntimeException(e); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/search/SearchConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/search/SearchConfig.java index c0364752e7a..2fa4cdea5dd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/search/SearchConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/search/SearchConfig.java @@ -7,7 +7,7 @@ import io.dropwizard.core.setup.Environment; @CPSBase -@JsonTypeInfo(property = "type", use = JsonTypeInfo.Id.CUSTOM ) +@JsonTypeInfo(property = "type", use = JsonTypeInfo.Id.CUSTOM) public interface SearchConfig { SearchProcessor createSearchProcessor(Environment environment, DatasetId id); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/search/solr/FilterValueConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/search/solr/FilterValueConfig.java index fb5706c8a29..3e605e1ea78 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/search/solr/FilterValueConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/search/solr/FilterValueConfig.java @@ -1,12 +1,12 @@ package com.bakdata.conquery.models.config.search.solr; -import java.util.Collections; -import java.util.Map; -import javax.annotation.CheckForNull; import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; +import java.util.Collections; +import java.util.Map; +import javax.annotation.CheckForNull; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.util.search.solr.entities.SolrFrontendValue; @@ -20,55 +20,55 @@ public class FilterValueConfig { /** - * Label for the special empty value to filter for empty entries. - */ - @NotNull - private String emptyLabel = "No Value"; - - /** - * Effectively the query that is sent to solr after we split the users search phrase into terms on whitespaces and join them together again after template resolving. - * Joining involves a boolean operator, so parentheses might be needed. - * The format string only gets a single argument, so refer to the argument using ${term}. The template is interpreted by {@link org.apache.commons.text.StringSubstitutor}. - *

- * An incoming search string is split on white spaces into terms. These terms are each (${term}) escaped and applied to this template and then concatenated with an AND . - * If you have a complex expression, consider to surround your template with parentheses, so the concatenating "AND"s apply to the whole template. - */ - @NotEmpty - private String queryTemplate = "( ${term} value_s:\"${term}\"^100 )"; + * Label for the special empty value to filter for empty entries. + */ + @NotNull + private String emptyLabel = "No Value"; - /** - * Determines the field by which the results of the default query (no user input, just results) are sorted. - * This is only possible for fields that are sortable (docValues or indexed). - */ - @CheckForNull - private String defaultSearchSortField = SolrFrontendValue.Fields.value_s; + /** + * Effectively the query that is sent to solr after we split the users search phrase into terms on whitespaces and join them together again after template resolving. + * Joining involves a boolean operator, so parentheses might be needed. + * The format string only gets a single argument, so refer to the argument using ${term}. The template is interpreted by {@link org.apache.commons.text.StringSubstitutor}. + *

+ * An incoming search string is split on white spaces into terms. These terms are each (${term}) escaped and applied to this template and then concatenated with an AND . + * If you have a complex expression, consider to surround your template with parentheses, so the concatenating "AND"s apply to the whole template. + */ + @NotEmpty + private String queryTemplate = "( ${term} value_s:\"${term}\"^100 )"; - private boolean sharedDocumentsOnSecondaryId = false; + /** + * Determines the field by which the results of the default query (no user input, just results) are sorted. + * This is only possible for fields that are sortable (docValues or indexed). + */ + @CheckForNull + private String defaultSearchSortField = SolrFrontendValue.Fields.value_s; - /** - * By default, for each value in a column a solr document is created. The id of this solr-document uses the column id and the column value. - * This can cause a large number of documents, many referring to the same value. - * When this map is populated, not the column id but a group name is used to form the document id, hence abstracting over all columns that point to the same group. - *

- * Beware that you need to ensure that equally named columns used in filters are also based on the same set of values. Otherwise, you might encounter unexpected - * query results. + private boolean sharedDocumentsOnSecondaryId = false; - * TableName -> ColumnName -> GroupName - * - * { - * "zoo_table": { - * "animal": "animals", - * "sponsor": "proteges" - * }, - * "pet_table": { - * "pet": "animals", - * "owner": "proteges" - * } - * } - * - */ - private Map<@NotBlank String,Map<@NotBlank String,@NotBlank String>> sharedColumnDocuments = Collections.emptyMap(); + /** + * By default, for each value in a column a solr document is created. The id of this solr-document uses the column id and the column value. + * This can cause a large number of documents, many referring to the same value. + * When this map is populated, not the column id but a group name is used to form the document id, hence abstracting over all columns that point to the same group. + *

+ * Beware that you need to ensure that equally named columns used in filters are also based on the same set of values. Otherwise, you might encounter unexpected + * query results. + * TableName -> ColumnName -> GroupName + * + * { + * "zoo_table": { + * "animal": "animals", + * "sponsor": "proteges" + * }, + * "pet_table": { + * "pet": "animals", + * "owner": "proteges" + * } + * } + * + */ + private Map<@NotBlank String, Map<@NotBlank String, @NotBlank String>> sharedColumnDocuments = Collections + .emptyMap(); /** @@ -79,25 +79,25 @@ public class FilterValueConfig { @Min(1) private int updateChunkSize = 100; - /** - * Returns the shared group for the column, if documents should be shared. - * - * @param column The column id that might belong to a group - * @return The group name or null. - */ - @JsonIgnore - public String getColumnGroup(Column column) { - String tableName = column.getTable().getName(); - String columnName = column.getName(); - String groupName = sharedColumnDocuments.getOrDefault(tableName, Collections.emptyMap()).get(columnName); - if (groupName != null) { - return groupName; - } + /** + * Returns the shared group for the column, if documents should be shared. + * + * @param column The column id that might belong to a group + * @return The group name or null. + */ + @JsonIgnore + public String getColumnGroup(Column column) { + String tableName = column.getTable().getName(); + String columnName = column.getName(); + String groupName = sharedColumnDocuments.getOrDefault(tableName, Collections.emptyMap()).get(columnName); + if (groupName != null) { + return groupName; + } - if (sharedDocumentsOnSecondaryId && column.getSecondaryId() != null) { - groupName = column.getSecondaryId().getName(); + if (sharedDocumentsOnSecondaryId && column.getSecondaryId() != null) { + groupName = column.getSecondaryId().getName(); - } - return groupName; - } -} \ No newline at end of file + } + return groupName; + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/search/solr/SolrConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/search/solr/SolrConfig.java index 958e57cce5c..388e1d83dfd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/search/solr/SolrConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/search/solr/SolrConfig.java @@ -1,9 +1,9 @@ package com.bakdata.conquery.models.config.search.solr; -import java.util.concurrent.TimeUnit; -import javax.annotation.Nullable; import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotNull; +import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.config.search.SearchConfig; @@ -48,23 +48,26 @@ public class SolrConfig implements SearchConfig { @Min(1) private int indexerClientQueueSize = 10; - /** - * Prefer http/1.1. - * This might help with network problems (e.g. reverse proxy). - * If unset, solr client tests a property see {@link HttpJdkSolrClient.Builder#useHttp1_1(boolean)}. - */ - @Nullable - private Boolean useHttp1_1; + /** + * Prefer http/1.1. + * This might help with network problems (e.g. reverse proxy). + * If unset, solr client tests a property see {@link HttpJdkSolrClient.Builder#useHttp1_1(boolean)}. + */ + @Nullable + private Boolean useHttp1_1; - @Override + @Override public SolrProcessor createSearchProcessor(Environment environment, DatasetId datasetId) { - SolrProcessor solrProcessor = new SolrProcessor(() -> createSearchClient(datasetId.getName()),() -> createIndexClient(datasetId.getName()), commitWithin, filterValue); + SolrProcessor solrProcessor = new SolrProcessor( + () -> createSearchClient(datasetId.getName()), + () -> createIndexClient(datasetId.getName()), + commitWithin, + filterValue); try { solrProcessor.start(); return solrProcessor; - } - catch (Exception e) { + } catch (Exception e) { throw new RuntimeException(e); } } @@ -72,9 +75,9 @@ public SolrProcessor createSearchProcessor(Environment environment, DatasetId da public SolrClient createSearchClient(@Nullable String collection) { log.info("Creating solr search client. Base url: {}, Collection: {})", baseSolrUrl, collection); - HttpJdkSolrClient.Builder builder = new HttpJdkSolrClient.Builder(baseSolrUrl) - .withConnectionTimeout(connectionTimeout.toSeconds(), TimeUnit.SECONDS) - .withRequestTimeout(requestTimeout.toSeconds(), TimeUnit.SECONDS); + HttpJdkSolrClient.Builder builder = new HttpJdkSolrClient.Builder(baseSolrUrl).withConnectionTimeout( + connectionTimeout.toSeconds(), + TimeUnit.SECONDS).withRequestTimeout(requestTimeout.toSeconds(), TimeUnit.SECONDS); if (collection != null) { builder.withDefaultCollection(collection); @@ -84,35 +87,37 @@ public SolrClient createSearchClient(@Nullable String collection) { builder.withBasicAuthCredentials(username, password); } - if (useHttp1_1 != null) { - builder.useHttp1_1(useHttp1_1); - } + if (useHttp1_1 != null) { + builder.useHttp1_1(useHttp1_1); + } - return builder.build(); + return builder.build(); } public SolrClient createIndexClient(@Nullable String collection) { log.info("Creating solr index client. Base url: {}, Collection: {})", baseSolrUrl, collection); - Http2SolrClient.Builder http2Builder = new Http2SolrClient.Builder(baseSolrUrl) - .withConnectionTimeout(connectionTimeout.toSeconds(), TimeUnit.SECONDS) - .withRequestTimeout(requestTimeout.toSeconds(), TimeUnit.SECONDS); + Http2SolrClient.Builder http2Builder = new Http2SolrClient.Builder(baseSolrUrl).withConnectionTimeout( + connectionTimeout.toSeconds(), + TimeUnit.SECONDS).withRequestTimeout(requestTimeout.toSeconds(), TimeUnit.SECONDS); if (username != null) { http2Builder.withBasicAuthCredentials(username, password); } - if (useHttp1_1 != null) { - http2Builder.useHttp1_1(useHttp1_1); - } + if (useHttp1_1 != null) { + http2Builder.useHttp1_1(useHttp1_1); + } Http2SolrClient http2Client = http2Builder.build(); - ConcurrentUpdateHttp2SolrClient.Builder concurrentClientBuilder = new ConcurrentUpdateHttp2SolrClient.Builder(baseSolrUrl, http2Client) - .withDefaultCollection(collection) + ConcurrentUpdateHttp2SolrClient.Builder concurrentClientBuilder = new ConcurrentUpdateHttp2SolrClient.Builder( + baseSolrUrl, + http2Client).withDefaultCollection(collection) .withQueueSize(indexerClientQueueSize) - .withThreadCount(indexerClientThreads); + .withThreadCount( + indexerClientThreads); return concurrentClientBuilder.build(); @@ -126,8 +131,7 @@ protected Result check() throws Exception { String status = (String) response.get("status"); if ("healthy".equalsIgnoreCase(status) || "ok".equalsIgnoreCase(status)) { return Result.builder().healthy().build(); - } - else { + } else { return Result.unhealthy(status); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java index 5eec64d3324..ba5eeadbc5e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java @@ -1,8 +1,8 @@ package com.bakdata.conquery.models.datasets; +import jakarta.validation.constraints.NotNull; import javax.annotation.CheckForNull; import javax.annotation.Nullable; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.models.datasets.concepts.Searchable; import com.bakdata.conquery.models.events.MajorTypeId; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/Dataset.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/Dataset.java index cd4d5d0f782..55e0c198677 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/Dataset.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/Dataset.java @@ -79,9 +79,7 @@ public Table getAllIdsTable() { @Override public MutableInjectableValues inject(MutableInjectableValues mutableInjectableValues) { - return mutableInjectableValues - .add(Dataset.class, this) - .add(DatasetId.class, getId()); + return mutableInjectableValues.add(Dataset.class, this).add(DatasetId.class, getId()); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/PreviewConfig.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/PreviewConfig.java index b68dfe73084..3972c83af4a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/PreviewConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/PreviewConfig.java @@ -1,14 +1,14 @@ package com.bakdata.conquery.models.datasets; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; +import jakarta.ws.rs.core.UriBuilder; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; -import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.models.auth.entities.Subject; import com.bakdata.conquery.models.common.Range; @@ -105,21 +105,42 @@ public record InfoCardSelect(@NotNull String label, SelectId select, String desc /** * Defines a group of selects that will be evaluated per quarter and year in the requested period of the entity-preview. */ - public record TimeStratifiedSelects(@NotNull String label, String description, @NotEmpty List selects) { + public record TimeStratifiedSelects( + @NotNull String label, + String description, + @NotEmpty List selects) { } @ValidationMethod(message = "Selects may be referenced only once.") @JsonIgnore public boolean isSelectsUnique() { - return timeStratifiedSelects.stream().map(TimeStratifiedSelects::selects).flatMap(Collection::stream).map(InfoCardSelect::select).distinct().count() - == timeStratifiedSelects.stream().map(TimeStratifiedSelects::selects).mapToLong(Collection::size).sum(); + return timeStratifiedSelects.stream() + .map(TimeStratifiedSelects::selects) + .flatMap(Collection::stream) + .map( + InfoCardSelect::select) + .distinct() + .count() == timeStratifiedSelects.stream() + .map( + TimeStratifiedSelects::selects) + .mapToLong(Collection::size) + .sum(); } @ValidationMethod(message = "Labels must be unique.") @JsonIgnore public boolean isLabelsUnique() { - return timeStratifiedSelects.stream().map(TimeStratifiedSelects::selects).flatMap(Collection::stream).map(InfoCardSelect::label).distinct().count() - == timeStratifiedSelects.stream().map(TimeStratifiedSelects::selects).mapToLong(Collection::size).sum(); + return timeStratifiedSelects.stream() + .map(TimeStratifiedSelects::selects) + .flatMap(Collection::stream) + .map( + InfoCardSelect::label) + .distinct() + .count() == timeStratifiedSelects.stream() + .map( + TimeStratifiedSelects::selects) + .mapToLong(Collection::size) + .sum(); } @JsonIgnore @@ -143,7 +164,11 @@ public boolean isSelectsLabelsDistinct() { @JsonIgnore @ValidationMethod(message = "timeStratifiedSelects' labels must be unique.") public boolean isStratifiedInfosUnique() { - return timeStratifiedSelects.stream().map(TimeStratifiedSelects::label).distinct().count() == timeStratifiedSelects.size(); + return timeStratifiedSelects.stream() + .map( + TimeStratifiedSelects::label) + .distinct() + .count() == timeStratifiedSelects.size(); } @@ -171,9 +196,7 @@ public String resolveSelectLabel(SelectResultInfo info) { */ @JsonIgnore public List getSelects() { - return getInfoCardSelects().stream() - .map(InfoCardSelect::select) - .collect(Collectors.toList()); + return getInfoCardSelects().stream().map(InfoCardSelect::select).collect(Collectors.toList()); } public ConnectorId resolveSearchConnector() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java index 16cbdfdc285..97893cba320 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java @@ -1,13 +1,13 @@ package com.bakdata.conquery.models.datasets; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; import java.util.Arrays; import java.util.HashSet; import java.util.Set; import java.util.stream.Stream; import javax.annotation.CheckForNull; import javax.annotation.Nullable; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.io.jackson.Initializing; import com.bakdata.conquery.io.jackson.View; @@ -41,7 +41,7 @@ @Slf4j @JsonDeserialize(converter = Table.Initializer.class) @ValidSqlTable(groups = {ValidationMode.Local.class}) -@EqualsAndHashCode(callSuper=false) +@EqualsAndHashCode(callSuper = false) public class Table extends LabeledNamespaceIdentifiable implements Initializing { /** @@ -112,16 +112,15 @@ public TableId createId() { public Stream findImports(NamespacedStorage storage) { final TableId thisId = getId(); - return storage.getAllImports() - .filter(imp -> imp.getTable().equals(thisId)) - .map(ImportId::resolve); + return storage.getAllImports().filter(imp -> imp.getTable().equals(thisId)).map(ImportId::resolve); } public Column getColumnByName(@NotNull String columnName) { return Arrays.stream(columns) - .filter(column -> column.getName().equals(columnName)) - .findFirst() - .orElseThrow(() -> new IllegalStateException(String.format("Column %s not found", columnName))); + .filter(column -> column.getName().equals(columnName)) + .findFirst() + .orElseThrow( + () -> new IllegalStateException(String.format("Column %s not found", columnName))); } /** diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java index f035a2163b2..de298cf5402 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java @@ -1,11 +1,11 @@ package com.bakdata.conquery.models.datasets.concepts; +import jakarta.validation.Valid; import java.util.Collections; import java.util.List; import java.util.Set; import java.util.stream.Collectors; import javax.annotation.Nullable; -import jakarta.validation.Valid; import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.io.jackson.Initializing; @@ -87,9 +87,11 @@ public void init() throws Exception { @JsonIgnore public List getDefaultSelects() { - return getSelects().stream().filter(Select::isDefault) - .map(select -> (SelectId) select.getId()) - .collect(Collectors.toList()); + return getSelects().stream() + .filter(Select::isDefault) + .map(select -> (SelectId) select.getId()) + .collect( + Collectors.toList()); } public abstract List getSelects(); @@ -132,16 +134,15 @@ public int countElements() { * Allows concepts to create their own altered FiltersNode if necessary. */ public QPNode createConceptQuery( - QueryPlanContext context, - List> filters, - List> aggregators, - EventDateUnionAggregator eventDateAggregators, - ValidityDate validityDate) { + QueryPlanContext context, + List> filters, + List> aggregators, + EventDateUnionAggregator eventDateAggregators, + ValidityDate validityDate) { final QPNode child; if (filters.isEmpty() && aggregators.isEmpty()) { child = new Leaf(); - } - else { + } else { child = FiltersNode.create(filters, aggregators, eventDateAggregators); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/ConceptElement.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/ConceptElement.java index 7c6575e8d6f..42d03af5984 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/ConceptElement.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/ConceptElement.java @@ -18,8 +18,7 @@ @Setter @Getter @EqualsAndHashCode(callSuper = true) -public abstract class ConceptElement> - extends LabeledNamespaceIdentifiable { +public abstract class ConceptElement> extends LabeledNamespaceIdentifiable { private String description; private List additionalInfos = Collections.emptyList(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Connector.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Connector.java index 4d25680db45..478e4b98635 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Connector.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Connector.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.datasets.concepts; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -7,8 +9,6 @@ import java.util.stream.Collectors; import javax.annotation.CheckForNull; import javax.annotation.Nullable; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.datasets.concepts.conditions.CTCondition; @@ -94,9 +94,7 @@ public Collection> getFilters() { @JsonIgnore public List { List selects); + Concept findConcept(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/StructureNode.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/StructureNode.java index 033cd358743..3ec9861aa7f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/StructureNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/StructureNode.java @@ -1,11 +1,11 @@ package com.bakdata.conquery.models.datasets.concepts; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.stream.Stream; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.apiv1.KeyValue; import com.bakdata.conquery.models.identifiable.LabeledNamespaceIdentifiable; @@ -19,7 +19,9 @@ import lombok.Setter; import lombok.ToString; -@Getter @Setter @ToString(callSuper=true,of={"children"}) +@Getter +@Setter +@ToString(callSuper = true, of = {"children"}) public class StructureNode extends LabeledNamespaceIdentifiable { public static final String MANAGED_STRUCTURE_STRUCTURE = "structure_structure"; @@ -28,7 +30,8 @@ public class StructureNode extends LabeledNamespaceIdentifiable @NotNull private DatasetId dataset; private String description; - @Valid @JsonManagedReference(MANAGED_STRUCTURE_STRUCTURE) + @Valid + @JsonManagedReference(MANAGED_STRUCTURE_STRUCTURE) private List children = Collections.emptyList(); @JsonBackReference(MANAGED_STRUCTURE_STRUCTURE) @EqualsAndHashCode.Exclude diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/ValidityDate.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/ValidityDate.java index 1f244494c96..96043798277 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/ValidityDate.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/ValidityDate.java @@ -56,7 +56,7 @@ public static ValidityDate create(Column startColumn, Column endColumn) { @CheckForNull public CDateRange getValidityDate(int event, Bucket bucket) { - if (extractor == null){ + if (extractor == null) { //TODO this is just a workaround: We should actually be using Initializing, which sadly gives us issues with LoadingUtil init(); } @@ -72,9 +72,11 @@ public boolean containsColumn(ColumnId id) { @ValidationMethod(message = "ValidityDate is not for Connectors' Table.") public boolean isForConnectorsTable() { - final boolean anyColumnNotForConnector = - (startColumn != null && !startColumn.getTable().equals(connector.resolveTableId())) - || (endColumn != null && !endColumn.getTable().equals(connector.resolveTableId())); + final boolean anyColumnNotForConnector = (startColumn != null && !startColumn.getTable() + .equals( + connector.resolveTableId())) || (endColumn != null && !endColumn.getTable() + .equals( + connector.resolveTableId())); final boolean columnNotForConnector = column != null && !column.getTable().equals(connector.resolveTableId()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/AndCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/AndCondition.java index 6e06ea5118c..76a7dd41920 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/AndCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/AndCondition.java @@ -1,10 +1,10 @@ package com.bakdata.conquery.models.datasets.concepts.conditions; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; import java.util.Collections; import java.util.List; import java.util.Map; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotEmpty; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; @@ -28,7 +28,9 @@ public class AndCondition implements CTCondition { private List conditions; @Override - public boolean matches(String value, CalculatedValue> rowMap) throws ConceptConfigurationException { + public boolean matches( + String value, + CalculatedValue> rowMap) throws ConceptConfigurationException { for (CTCondition cond : conditions) { if (!cond.matches(value, rowMap)) { return false; @@ -47,17 +49,21 @@ public void init(ConceptElement node) throws ConceptConfigurationException { @Override public WhereCondition convertToSqlCondition(CTConditionContext context) { return conditions.stream() - .map(condition -> condition.convertToSqlCondition(context)) - .reduce(WhereCondition::and) - .orElseThrow( - () -> new IllegalStateException("At least one condition is required to convert %s to a SQL condition.".formatted(getClass())) - ); + .map(condition -> condition.convertToSqlCondition(context)) + .reduce( + WhereCondition::and) + .orElseThrow( + () -> new IllegalStateException( + "At least one condition is required to convert %s to a SQL condition.".formatted(getClass())) + ); } @Override public ConceptConditions buildExpression(CTConditionContext context, ConceptElement id) { - List conceptConditions = conditions.stream().map(cond -> cond.buildExpression(context, id)) - .toList(); + List conceptConditions = conditions.stream() + .map( + cond -> cond.buildExpression(context, id)) + .toList(); ConceptConditions out = new ConceptConditions(id, Collections.emptyMap()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/CTCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/CTCondition.java index 241031bf34b..dc6a0a3dddf 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/CTCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/CTCondition.java @@ -65,12 +65,20 @@ public ConceptConditions and(ConceptConditions other) { return this; } - Set fields = Stream.of(other.conditions.keySet(), conditions.keySet()).flatMap(Collection::stream) - .map(Field::getUnqualifiedName) - .collect(Collectors.toSet()); - - Map> rByName = other.conditions.keySet().stream().collect(Collectors.toMap(Field::getUnqualifiedName, Function.identity())); - Map> lByName = conditions.keySet().stream().collect(Collectors.toMap(Field::getUnqualifiedName, Function.identity())); + Set fields = Stream.of(other.conditions.keySet(), conditions.keySet()) + .flatMap( + Collection::stream) + .map(Field::getUnqualifiedName) + .collect(Collectors.toSet()); + + Map> rByName = other.conditions.keySet() + .stream() + .collect( + Collectors.toMap(Field::getUnqualifiedName, Function.identity())); + Map> lByName = conditions.keySet() + .stream() + .collect( + Collectors.toMap(Field::getUnqualifiedName, Function.identity())); Map, FieldCondition> combined = new HashMap<>(); @@ -93,15 +101,18 @@ public ConceptConditions and(ConceptConditions other) { FieldCondition otherCond = other.conditions.get(rField); FieldCondition myCond = conditions.get(lField); - FieldCondition condition = new FieldCondition(myCond.extractor, Sets.intersection(otherCond.params, myCond.params)); + FieldCondition condition = new FieldCondition( + myCond.extractor, + Sets.intersection(otherCond.params, myCond.params)); // Recompute string field to fit in all values. // Otherwise, assume both fields are the same type Field outField; if (lField.getDataType().isString()) { - outField = field(lField.getUnqualifiedName(), VARCHAR(Math.max(lField.getDataType().length(), rField.getDataType().length()))); - } - else { + outField = field( + lField.getUnqualifiedName(), + VARCHAR(Math.max(lField.getDataType().length(), rField.getDataType().length()))); + } else { assert lField.getDataType().equals(rField.getDataType()); outField = lField; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/ColumnEqualCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/ColumnEqualCondition.java index 1f8e49dc437..5cdb6b7a774 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/ColumnEqualCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/ColumnEqualCondition.java @@ -4,13 +4,12 @@ import static org.jooq.impl.DSL.name; import static org.jooq.impl.SQLDataType.VARCHAR; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotEmpty; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotEmpty; - import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; @@ -68,8 +67,12 @@ private int fieldLength() { @Override public ConceptConditions buildExpression(CTConditionContext context, ConceptElement id) { - FieldCondition condition = new FieldCondition(field(name(getColumn()), VARCHAR), values.stream().map(DSL::inline).collect(Collectors.toSet())); + FieldCondition condition = new FieldCondition( + field(name(getColumn()), VARCHAR), + values.stream().map(DSL::inline).collect(Collectors.toSet())); - return new ConceptConditions(id, Map.of(field(name("%s_equal".formatted(column)), VARCHAR(fieldLength())), condition)); + return new ConceptConditions( + id, + Map.of(field(name("%s_equal".formatted(column)), VARCHAR(fieldLength())), condition)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/EqualCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/EqualCondition.java index 5afdfae99c7..e60e35ab08e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/EqualCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/EqualCondition.java @@ -4,10 +4,10 @@ import static org.jooq.impl.DSL.name; import static org.jooq.impl.SQLDataType.VARCHAR; +import jakarta.validation.constraints.NotEmpty; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import jakarta.validation.constraints.NotEmpty; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; @@ -46,7 +46,10 @@ public boolean matches(String value, CalculatedValue> rowMap @Override public WhereCondition convertToSqlCondition(CTConditionContext context) { - return new MultiSelectCondition(field(context.getConnectorColumn(), VARCHAR), values.toArray(String[]::new), context.getFunctionProvider()); + return new MultiSelectCondition( + field(context.getConnectorColumn(), VARCHAR), + values.toArray(String[]::new), + context.getFunctionProvider()); } private int fieldLength() { @@ -55,7 +58,11 @@ private int fieldLength() { @Override public ConceptConditions buildExpression(CTConditionContext context, ConceptElement id) { - FieldCondition condition = new FieldCondition(field(name(context.getConnectorColumn()), VARCHAR), values.stream().map(DSL::inline).collect(Collectors.toSet())); - return new ConceptConditions(id, Map.of(field(name(CTConditionContext.COLUMN_VALUE_FIELD), VARCHAR(fieldLength())), condition)); + FieldCondition condition = new FieldCondition( + field(name(context.getConnectorColumn()), VARCHAR), + values.stream().map(DSL::inline).collect(Collectors.toSet())); + return new ConceptConditions( + id, + Map.of(field(name(CTConditionContext.COLUMN_VALUE_FIELD), VARCHAR(fieldLength())), condition)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/GroovyCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/GroovyCondition.java index 13bbb04a455..d4f989f68c0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/GroovyCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/GroovyCondition.java @@ -1,9 +1,9 @@ package com.bakdata.conquery.models.datasets.concepts.conditions; +import jakarta.validation.constraints.NotEmpty; import java.time.LocalDate; import java.util.Map; import java.util.stream.Stream; -import jakarta.validation.constraints.NotEmpty; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.common.Range; @@ -30,8 +30,8 @@ public class GroovyCondition implements CTCondition { public static final String[] AUTO_IMPORTS = Stream.of( - LocalDate.class, - Range.class + LocalDate.class, + Range.class ).map(Class::getName).toArray(String[]::new); @Getter @@ -59,26 +59,30 @@ private void compile() throws ConceptConfigurationException { compiled = (ConditionScript) groovy.parse(script); compiled.setNode(node); - } - catch (Exception | Error e) { + } catch (Exception | Error e) { throw new ConceptConfigurationException(node, "Failed to compile condition '" + script + "'", e); } } } @Override - public boolean matches(String value, CalculatedValue> rowMap) throws ConceptConfigurationException { + public boolean matches( + String value, + CalculatedValue> rowMap) throws ConceptConfigurationException { try { return compiled.matches(rowMap.getValue(), value); - } - catch (Exception e) { - throw new ConceptConfigurationException(node, "Could not execute condition \"" + script + "\" on " + rowMap.getValue(), e); + } catch (Exception e) { + throw new ConceptConfigurationException( + node, + "Could not execute condition \"" + script + "\" on " + rowMap.getValue(), + e); } } @Override public WhereCondition convertToSqlCondition(CTConditionContext context) { - throw new UnsupportedOperationException("SQL conversion of CTCondition %s not supported yet.".formatted(getClass())); + throw new UnsupportedOperationException( + "SQL conversion of CTCondition %s not supported yet.".formatted(getClass())); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsEmptyCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsEmptyCondition.java index cc967ab09cb..410b5b612e0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsEmptyCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsEmptyCondition.java @@ -16,7 +16,6 @@ import lombok.NonNull; import lombok.Setter; import org.jooq.Condition; -import org.jooq.impl.DSL; /** * This condition requires that the selected Column has a value. @@ -43,7 +42,9 @@ public WhereCondition convertToSqlCondition(CTConditionContext context) { @Override public ConceptConditions buildExpression(CTConditionContext context, ConceptElement id) { - FieldCondition condition = new FieldCondition(context.getFunctionProvider().isNull(field(name(column))), Set.of(inline(true))); + FieldCondition condition = new FieldCondition( + context.getFunctionProvider().isNull(field(name(column))), + Set.of(inline(true))); return new ConceptConditions(id, Map.of(field(name("%s_is_empty".formatted(column)), BOOLEAN), condition)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsPresentCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsPresentCondition.java index 5a1ffb77464..6217802bbe5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsPresentCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsPresentCondition.java @@ -2,7 +2,6 @@ import static org.jooq.impl.DSL.*; import static org.jooq.impl.SQLDataType.BOOLEAN; -import static org.jooq.impl.SQLDataType.VARCHAR; import java.util.Map; import java.util.Set; @@ -17,7 +16,6 @@ import lombok.NonNull; import lombok.Setter; import org.jooq.Condition; -import org.jooq.impl.DSL; /** * This condition requires that the selected Column has a value. @@ -44,7 +42,9 @@ public WhereCondition convertToSqlCondition(CTConditionContext context) { @Override public ConceptConditions buildExpression(CTConditionContext context, ConceptElement id) { - FieldCondition condition = new FieldCondition(context.getFunctionProvider().isNull(field(name(column))), Set.of(inline(false))); + FieldCondition condition = new FieldCondition( + context.getFunctionProvider().isNull(field(name(column))), + Set.of(inline(false))); return new ConceptConditions(id, Map.of(field(name("%s_is_empty".formatted(column)), BOOLEAN), condition)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/NotCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/NotCondition.java index 3dfb4d902e4..e62ef9409bf 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/NotCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/NotCondition.java @@ -1,7 +1,7 @@ package com.bakdata.conquery.models.datasets.concepts.conditions; -import java.util.Map; import jakarta.validation.Valid; +import java.util.Map; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; @@ -15,15 +15,19 @@ /** * This condition matches if its child does not. */ -@CPSType(id="NOT", base=CTCondition.class) +@CPSType(id = "NOT", base = CTCondition.class) @Deprecated public class NotCondition implements CTCondition { - @Setter @Getter @Valid + @Setter + @Getter + @Valid private CTCondition condition; @Override - public boolean matches(String value, CalculatedValue> rowMap) throws ConceptConfigurationException { + public boolean matches( + String value, + CalculatedValue> rowMap) throws ConceptConfigurationException { return !condition.matches(value, rowMap); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixCondition.java index 28a72733e22..f4418a53ce9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixCondition.java @@ -3,6 +3,7 @@ import static org.jooq.impl.DSL.field; import static org.jooq.impl.SQLDataType.VARCHAR; +import jakarta.validation.constraints.NotEmpty; import java.util.Arrays; import java.util.Map; import java.util.stream.Collectors; @@ -10,10 +11,9 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.sql.conversion.model.filter.ConditionWrappingWhereCondition; +import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.util.CalculatedValue; -import jakarta.validation.constraints.NotEmpty; import lombok.Getter; import lombok.Setter; import lombok.ToString; @@ -44,8 +44,13 @@ public boolean matches(String value, CalculatedValue> rowMap @Override public WhereCondition convertToSqlCondition(CTConditionContext context) { - String pattern = Arrays.stream(prefixes).collect(Collectors.joining("|", "", context.getFunctionProvider().getAnyCharRegex())); - Condition condition = context.getFunctionProvider().likeRegex(field(context.getConnectorColumn(), VARCHAR), pattern); + String pattern = Arrays.stream(prefixes) + .collect( + Collectors.joining("|", "", context.getFunctionProvider().getAnyCharRegex())); + Condition condition = context.getFunctionProvider() + .likeRegex( + field(context.getConnectorColumn(), VARCHAR), + pattern); return new ConditionWrappingWhereCondition(condition); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixRangeCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixRangeCondition.java index c8a653828c6..5464faffcca 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixRangeCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixRangeCondition.java @@ -3,15 +3,15 @@ import static org.jooq.impl.DSL.field; import static org.jooq.impl.SQLDataType.VARCHAR; -import java.util.Map; import jakarta.validation.constraints.NotEmpty; +import java.util.Map; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.sql.conversion.model.filter.ConditionWrappingWhereCondition; +import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.util.CalculatedValue; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; @@ -73,8 +73,7 @@ private String buildSqlRegexPattern(SqlFunctionProvider functionProvider) { char maxChar = maxChars[i]; if (minChar != maxChar) { builder.append("[%s-%s]".formatted(minChar, maxChar)); - } - else { + } else { builder.append(minChar); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/Filter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/Filter.java index 5f9ee9f52ed..74841d63fc3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/Filter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/Filter.java @@ -56,21 +56,26 @@ public DatasetId getDataset() { return getConnector().getDataset(); } - public FrontendFilterConfiguration.Top createFrontendConfig(ConqueryConfig conqueryConfig) throws ConceptConfigurationException { + public FrontendFilterConfiguration.Top createFrontendConfig( + ConqueryConfig conqueryConfig) throws ConceptConfigurationException { final FrontendFilterConfiguration.Top f = FrontendFilterConfiguration.Top.builder() - .id(getId()) - .label(getLabel()) - .tooltip(getTooltip()) - .unit(getUnit()) - .allowDropFile(getAllowDropFile()) - .pattern(getPattern()) - .defaultValue(getDefaultValue()) - .build(); + .id(getId()) + .label( + getLabel()) + .tooltip(getTooltip()) + .unit(getUnit()) + .allowDropFile(getAllowDropFile()) + .pattern( + getPattern()) + .defaultValue(getDefaultValue()) + .build(); configureFrontend(f, conqueryConfig); return f; } - protected abstract void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig conqueryConfig) throws ConceptConfigurationException; + protected abstract void configureFrontend( + FrontendFilterConfiguration.Top f, + ConqueryConfig conqueryConfig) throws ConceptConfigurationException; public abstract FilterNode createFilterNode(FILTER_VALUE filterValue); @@ -85,7 +90,12 @@ public boolean isForConnectorsTable() { continue; } - log.error("Filter[{}] of Table[{}] is not of Connector[{}]#Table[{}]", getId(), column.getTable(), connector.getId(), tableId); + log.error( + "Filter[{}] of Table[{}] is not of Connector[{}]#Table[{}]", + getId(), + column.getTable(), + connector.getId(), + tableId); valid = false; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/GroupFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/GroupFilter.java index 5ba69b2190c..f8b38f52c1c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/GroupFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/GroupFilter.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.models.datasets.concepts.filters; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.type.TypeFactory; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/AbstractSelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/AbstractSelectFilter.java index e69de29bb2d..8b137891791 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/AbstractSelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/AbstractSelectFilter.java @@ -0,0 +1 @@ + diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountFilter.java index d73f3b9e016..246fb79e0ba 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountFilter.java @@ -1,9 +1,9 @@ package com.bakdata.conquery.models.datasets.concepts.filters.specific; +import jakarta.validation.constraints.NotNull; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.apiv1.frontend.FrontendFilterConfiguration; import com.bakdata.conquery.apiv1.frontend.FrontendFilterType; @@ -51,10 +51,16 @@ public void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig } if (distinctByColumn != null && !getDistinctByColumn().isEmpty()) { - return new RangeFilterNode(value, new DistinctValuesWrapperAggregator(new CountAggregator(getColumn().resolve()), getDistinctByColumn().stream().map(ColumnId::resolve).toList())); + return new RangeFilterNode( + value, + new DistinctValuesWrapperAggregator( + new CountAggregator(getColumn().resolve()), + getDistinctByColumn().stream().map(ColumnId::resolve).toList())); } - return new RangeFilterNode(value, new DistinctValuesWrapperAggregator(new CountAggregator(), List.of(getColumn().resolve()))); + return new RangeFilterNode( + value, + new DistinctValuesWrapperAggregator(new CountAggregator(), List.of(getColumn().resolve()))); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountQuartersFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountQuartersFilter.java index a985f4e0468..269aba930f0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountQuartersFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountQuartersFilter.java @@ -1,33 +1,27 @@ package com.bakdata.conquery.models.datasets.concepts.filters.specific; + import java.util.List; -import java.util.Set; import javax.annotation.Nullable; import com.bakdata.conquery.apiv1.frontend.FrontendFilterConfiguration; import com.bakdata.conquery.apiv1.frontend.FrontendFilterType; import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.models.common.ColumnUtils; import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.DaterangeSelectOrFilter; import com.bakdata.conquery.models.datasets.concepts.filters.AggregationFilter; import com.bakdata.conquery.models.datasets.concepts.filters.Filter; -import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; import com.bakdata.conquery.models.query.filter.RangeFilterNode; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.CountQuartersOfDatesAggregator; import com.bakdata.conquery.models.query.queryplan.filter.AggregationFilterNode; import com.bakdata.conquery.sql.conversion.model.aggregator.CountQuartersSqlAggregator; import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; -import com.fasterxml.jackson.annotation.JsonIgnore; -import io.dropwizard.validation.ValidationMethod; import lombok.Getter; import lombok.Setter; -import static com.bakdata.conquery.models.events.MajorTypeId.DATE_RANGE; - @Setter @Getter @CPSType(id = "COUNT_QUARTERS", base = Filter.class) diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DateDistanceFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DateDistanceFilter.java index 3418d6d0ca5..6dc4e3b5559 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DateDistanceFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DateDistanceFilter.java @@ -1,5 +1,11 @@ package com.bakdata.conquery.models.datasets.concepts.filters.specific; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.Set; + import com.bakdata.conquery.apiv1.frontend.FrontendFilterConfiguration; import com.bakdata.conquery.apiv1.frontend.FrontendFilterType; import com.bakdata.conquery.io.cps.CPSType; @@ -17,16 +23,10 @@ import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotNull; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import java.time.temporal.ChronoUnit; -import java.util.List; -import java.util.Set; - /** * This filter represents a select in the front end. This means that the user can select one or more values from a list of values. */ @@ -55,10 +55,14 @@ public boolean isValidColumnType() { } @Override - public void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig conqueryConfig) throws ConceptConfigurationException { + public void configureFrontend( + FrontendFilterConfiguration.Top f, + ConqueryConfig conqueryConfig) throws ConceptConfigurationException { MajorTypeId type = getColumn().resolve().getType(); if (type != MajorTypeId.DATE) { - throw new ConceptConfigurationException(getConnector(), "DATE_DISTANCE filter is incompatible with columns of type " + type); + throw new ConceptConfigurationException( + getConnector(), + "DATE_DISTANCE filter is incompatible with columns of type " + type); } f.setType(FrontendFilterType.Fields.INTEGER_RANGE); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DurationSumFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DurationSumFilter.java index a57964a527b..29fa17fd722 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DurationSumFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DurationSumFilter.java @@ -1,23 +1,18 @@ package com.bakdata.conquery.models.datasets.concepts.filters.specific; +import jakarta.validation.Valid; import java.util.ArrayList; import java.util.List; -import java.util.Set; import javax.annotation.Nullable; -import com.fasterxml.jackson.annotation.JsonAlias; -import jakarta.validation.Valid; - import com.bakdata.conquery.apiv1.frontend.FrontendFilterConfiguration; import com.bakdata.conquery.apiv1.frontend.FrontendFilterType; import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.models.common.ColumnUtils; import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.concepts.DaterangeSelectOrFilter; import com.bakdata.conquery.models.datasets.concepts.filters.AggregationFilter; import com.bakdata.conquery.models.datasets.concepts.filters.Filter; -import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.exceptions.ConceptConfigurationException; import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; import com.bakdata.conquery.models.query.filter.RangeFilterNode; @@ -28,8 +23,8 @@ import com.bakdata.conquery.models.query.queryplan.filter.AggregationFilterNode; import com.bakdata.conquery.sql.conversion.model.aggregator.DurationSumSqlAggregator; import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; +import com.fasterxml.jackson.annotation.JsonAlias; import com.fasterxml.jackson.annotation.JsonIgnore; -import io.dropwizard.validation.ValidationMethod; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.extern.slf4j.Slf4j; @@ -70,7 +65,9 @@ public List getRequiredColumns() { } @Override - public void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig conqueryConfig) throws ConceptConfigurationException { + public void configureFrontend( + FrontendFilterConfiguration.Top f, + ConqueryConfig conqueryConfig) throws ConceptConfigurationException { f.setType(FrontendFilterType.Fields.INTEGER_RANGE); f.setMin(0); } @@ -82,11 +79,13 @@ private boolean hasDistinct() { @Override public AggregationFilterNode createFilterNode(Range.LongRange value) { - ColumnAggregator aggregator = isSingleColumnDaterange() ? new DurationSumAggregator(getColumn().resolve()) - : new TwoColumnDurationSumAggregator(startColumn.resolve(), endColumn.resolve()); + ColumnAggregator aggregator = isSingleColumnDaterange() ? new DurationSumAggregator( + getColumn().resolve()) : new TwoColumnDurationSumAggregator(startColumn.resolve(), endColumn.resolve()); if (hasDistinct()) { - aggregator = new DistinctValuesWrapperAggregator<>(aggregator, distinctBy.stream().map(ColumnId::resolve).toList()); + aggregator = new DistinctValuesWrapperAggregator<>( + aggregator, + distinctBy.stream().map(ColumnId::resolve).toList()); } return new RangeFilterNode(value, aggregator); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java index 0da9a4d2510..279053d2062 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java @@ -41,7 +41,9 @@ public class FlagFilter extends EventFilter> { private final Map flags; @Override - protected void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig conqueryConfig) throws ConceptConfigurationException { + protected void configureFrontend( + FrontendFilterConfiguration.Top f, + ConqueryConfig conqueryConfig) throws ConceptConfigurationException { f.setType(FrontendFilterType.Fields.MULTI_SELECT); f.setOptions(flags.keySet().stream().map(key -> new FrontendValue(key, key)).toList()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java index ccbd0f7866e..88511d12f86 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java @@ -1,15 +1,14 @@ package com.bakdata.conquery.models.datasets.concepts.filters.specific; -import java.math.BigDecimal; -import java.util.List; - -import com.bakdata.conquery.models.common.ColumnUtils; import jakarta.validation.Valid; import jakarta.validation.constraints.NotNull; +import java.math.BigDecimal; +import java.util.List; import com.bakdata.conquery.apiv1.frontend.FrontendFilterConfiguration; import com.bakdata.conquery.apiv1.frontend.FrontendFilterType; import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.models.common.ColumnUtils; import com.bakdata.conquery.models.common.IRange; import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.config.ConqueryConfig; @@ -61,13 +60,17 @@ public List getRequiredColumns() { @Override - public void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig conqueryConfig) throws ConceptConfigurationException { + public void configureFrontend( + FrontendFilterConfiguration.Top f, + ConqueryConfig conqueryConfig) throws ConceptConfigurationException { final MajorTypeId typeId = getColumn().resolve().getType(); final String type = switch (typeId) { case MONEY -> FrontendFilterType.Fields.MONEY_RANGE; case INTEGER -> FrontendFilterType.Fields.INTEGER_RANGE; case DECIMAL, REAL -> FrontendFilterType.Fields.REAL_RANGE; - default -> throw new ConceptConfigurationException(getConnector(), "NUMBER filter is incompatible with columns of type " + typeId); + default -> throw new ConceptConfigurationException( + getConnector(), + "NUMBER filter is incompatible with columns of type " + typeId); }; f.setType(type); @@ -83,7 +86,8 @@ public EventFilterNode createFilterNode(RANGE value) { case INTEGER -> new IntegerFilterNode(column, (Range.LongRange) value); case DECIMAL -> new DecimalFilterNode(column, (Range) value); case REAL -> new RealFilterNode(column, Range.DoubleRange.fromNumberRange(value)); - default -> throw new IllegalStateException(String.format("Column type %s may not be used (Assignment should not have been possible)", column)); + default -> throw new IllegalStateException( + String.format("Column type %s may not be used (Assignment should not have been possible)", column)); }; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/PrefixTextFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/PrefixTextFilter.java index 314e390b76f..07e0279e2e8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/PrefixTextFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/PrefixTextFilter.java @@ -1,15 +1,14 @@ package com.bakdata.conquery.models.datasets.concepts.filters.specific; -import java.util.EnumSet; -import java.util.List; - -import com.bakdata.conquery.models.common.ColumnUtils; import jakarta.validation.Valid; import jakarta.validation.constraints.NotNull; +import java.util.EnumSet; +import java.util.List; import com.bakdata.conquery.apiv1.frontend.FrontendFilterConfiguration; import com.bakdata.conquery.apiv1.frontend.FrontendFilterType; import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.models.common.ColumnUtils; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.concepts.filters.EventFilter; import com.bakdata.conquery.models.datasets.concepts.filters.Filter; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/QuartersInYearFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/QuartersInYearFilter.java index 9aef5558bb0..97420ddd8a8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/QuartersInYearFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/QuartersInYearFilter.java @@ -1,15 +1,14 @@ package com.bakdata.conquery.models.datasets.concepts.filters.specific; -import java.util.EnumSet; -import java.util.List; - -import com.bakdata.conquery.models.common.ColumnUtils; import jakarta.validation.Valid; import jakarta.validation.constraints.NotNull; +import java.util.EnumSet; +import java.util.List; import com.bakdata.conquery.apiv1.frontend.FrontendFilterConfiguration; import com.bakdata.conquery.apiv1.frontend.FrontendFilterType; import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.models.common.ColumnUtils; import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.concepts.filters.AggregationFilter; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/QueryContextResolvable.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/QueryContextResolvable.java index 4a0682ec939..1128f6a4073 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/QueryContextResolvable.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/QueryContextResolvable.java @@ -17,4 +17,4 @@ public interface QueryContextResolvable { void resolve(QueryResolveContext context); -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java index 1d88688531c..d89e91d44ce 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java @@ -1,5 +1,14 @@ package com.bakdata.conquery.models.datasets.concepts.filters.specific; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import javax.annotation.CheckForNull; + import com.bakdata.conquery.apiv1.FilterTemplate; import com.bakdata.conquery.apiv1.LabelMap; import com.bakdata.conquery.apiv1.frontend.FrontendFilterConfiguration; @@ -20,20 +29,11 @@ import com.google.common.collect.BiMap; import com.google.common.collect.ImmutableBiMap; import io.dropwizard.validation.ValidationMethod; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotNull; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import javax.annotation.CheckForNull; -import java.util.ArrayList; -import java.util.EnumSet; -import java.util.List; -import java.util.function.Predicate; -import java.util.stream.Collectors; - @Setter @Getter @NoArgsConstructor @@ -68,7 +68,9 @@ public boolean isValidColumnType() { } @Override - public void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig conqueryConfig) throws ConceptConfigurationException { + public void configureFrontend( + FrontendFilterConfiguration.Top f, + ConqueryConfig conqueryConfig) throws ConceptConfigurationException { final SearchIndexId searchIndexId = getTemplate(); if (searchIndexId != null) { f.setTemplate((FilterTemplate) searchIndexId.resolve()); @@ -76,7 +78,10 @@ public void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig f.setType(getFilterType()); // If either not searches are available or all are disabled, we allow users to supply their own values - f.setCreatable(conqueryConfig.getFrontend().isAlwaysAllowCreateValue() || getSearchReferences().stream().noneMatch(Predicate.not(Searchable::isSearchDisabled))); + f.setCreatable( + conqueryConfig.getFrontend().isAlwaysAllowCreateValue() || getSearchReferences().stream() + .noneMatch( + Predicate.not(Searchable::isSearchDisabled))); f.setOptions(collectLabels()); } @@ -115,9 +120,11 @@ public List getSearchReferences() { @NotNull protected List collectLabels() { - return labels.entrySet().stream() - .map(entry -> new FrontendValue(entry.getKey(), entry.getValue())) - .collect(Collectors.toList()); + return labels.entrySet() + .stream() + .map(entry -> new FrontendValue(entry.getKey(), entry.getValue())) + .collect( + Collectors.toList()); } @JsonIgnore diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SingleSelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SingleSelectFilter.java index cb19b38917f..8c474d01583 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SingleSelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SingleSelectFilter.java @@ -23,7 +23,10 @@ public class SingleSelectFilter extends SelectFilter { @Override public EventFilterNode createFilterNode(String value) { if (getSubstringRange() != null && !getSubstringRange().isAll()) { - return new SubstringMultiSelectFilterNode(getColumn().resolve(), Collections.singleton(value), getSubstringRange()); + return new SubstringMultiSelectFilterNode( + getColumn().resolve(), + Collections.singleton(value), + getSubstringRange()); } return new MultiSelectFilterNode(getColumn().resolve(), Collections.singleton(value)); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java index 10e87702ac5..494471766e7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java @@ -1,5 +1,11 @@ package com.bakdata.conquery.models.datasets.concepts.filters.specific; +import jakarta.validation.constraints.NotNull; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import javax.annotation.Nullable; + import com.bakdata.conquery.apiv1.frontend.FrontendFilterConfiguration; import com.bakdata.conquery.apiv1.frontend.FrontendFilterType; import com.bakdata.conquery.io.cps.CPSType; @@ -29,17 +35,11 @@ import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; -import jakarta.validation.constraints.NotNull; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; -import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - /** * This filter represents a filter on the sum of one integer column. */ @@ -59,14 +59,18 @@ public class SumFilter> extends Aggreg private List distinctByColumn = Collections.emptyList(); @Override - public void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig conqueryConfig) throws ConceptConfigurationException { + public void configureFrontend( + FrontendFilterConfiguration.Top f, + ConqueryConfig conqueryConfig) throws ConceptConfigurationException { final MajorTypeId typeId = getColumn().resolve().getType(); final String type = switch (typeId) { case MONEY -> FrontendFilterType.Fields.MONEY_RANGE; case INTEGER -> FrontendFilterType.Fields.INTEGER_RANGE; case DECIMAL, REAL -> FrontendFilterType.Fields.REAL_RANGE; default -> - throw new ConceptConfigurationException(getConnector(), "NUMBER filter is incompatible with columns of type " + typeId); + throw new ConceptConfigurationException( + getConnector(), + "NUMBER filter is incompatible with columns of type " + typeId); }; f.setType(type); @@ -99,9 +103,11 @@ public List getRequiredColumns() { } if (distinctByColumn != null && !distinctByColumn.isEmpty()) { - return new RangeFilterNode(range, new DistinctValuesWrapperAggregator(getAggregator(), getDistinctByColumn().stream() - .map(ColumnId::resolve) - .toList())); + return new RangeFilterNode( + range, + new DistinctValuesWrapperAggregator( + getAggregator(), + getDistinctByColumn().stream().map(ColumnId::resolve).toList())); } return new RangeFilterNode(range, getAggregator()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java index 4a1828699e5..b25ccefe7b9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java @@ -71,9 +71,7 @@ public String getColumnName() { return getLabel(); } // The select originates from a connector and the corresponding concept has more than one connector -> Print also the connector - return ((Connector) getHolder()).getLabel() - + ' ' - + getLabel(); + return ((Connector) getHolder()).getLabel() + ' ' + getLabel(); } public SelectResultInfo getResultInfo(CQConcept cqConcept) { @@ -97,7 +95,12 @@ public boolean isForConnectorsTable() { continue; } - log.error("Select[{}] of Table[{}] is not of Connector[{}]#Table[{}]", getId(), column.getTable(), connector.getId(), connector.resolveTableId()); + log.error( + "Select[{}] of Table[{}] is not of Connector[{}]#Table[{}]", + getId(), + column.getTable(), + connector.getId(), + connector.resolveTableId()); valid = false; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java index c803aaea202..5cf0bb9e8da 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.models.datasets.concepts.select.concept; -import java.sql.ResultSet; -import java.sql.SQLException; import java.util.Set; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; @@ -50,7 +48,9 @@ public Aggregator createAggregator() { @Override public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printSettings) { if (isAsIds()) { - return printerFactory.getListPrinter(new ConceptIdPrinter((TreeConcept) getHolder().findConcept(), printSettings), printSettings); + return printerFactory.getListPrinter( + new ConceptIdPrinter((TreeConcept) getHolder().findConcept(), printSettings), + printSettings); } return printerFactory.getListPrinter(printerFactory.getStringPrinter(printSettings), printSettings); @@ -60,7 +60,10 @@ public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printS public SelectResultInfo getResultInfo(CQConcept cqConcept) { if (isAsIds()) { - return new SelectResultInfo(this, cqConcept, Set.of(new SemanticType.ConceptColumnT(cqConcept.getConceptId()))); + return new SelectResultInfo( + this, + cqConcept, + Set.of(new SemanticType.ConceptColumnT(cqConcept.getConceptId()))); } return new SelectResultInfo(this, cqConcept, Set.of(new SemanticType.ConceptColumnT(cqConcept.getConceptId()))); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDateUnionSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDateUnionSelect.java index 1b7d1f43a4c..e3fcb2a83e3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDateUnionSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDateUnionSelect.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.models.datasets.concepts.select.concept.specific; -import java.sql.ResultSet; -import java.sql.SQLException; import java.util.List; import java.util.stream.Collectors; @@ -28,11 +26,13 @@ public class EventDateUnionSelect extends UniversalSelect { public Aggregator createAggregator() { EventDateUnionAggregator dateUnionAggregator = new EventDateUnionAggregator(); - dateUnionAggregator.setRequiredTables(getHolder().findConcept() - .getConnectors() - .stream() - .map(Connector::getResolvedTable) - .collect(Collectors.toSet())); + dateUnionAggregator.setRequiredTables( + getHolder().findConcept() + .getConnectors() + .stream() + .map(Connector::getResolvedTable) + .collect( + Collectors.toSet())); return dateUnionAggregator; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDurationSumSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDurationSumSelect.java index d5bb67b5c58..8710858e1a2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDurationSumSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDurationSumSelect.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.models.datasets.concepts.select.concept.specific; -import java.sql.ResultSet; -import java.sql.SQLException; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.select.Select; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/ExistsSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/ExistsSelect.java index f02a41fd630..46e21b40806 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/ExistsSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/ExistsSelect.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.models.datasets.concepts.select.concept.specific; -import java.sql.ResultSet; -import java.sql.SQLException; import java.util.Set; import java.util.stream.Collectors; @@ -46,6 +44,12 @@ public ResultType getResultType() { } private Set

collectRequiredTables() { - return this.getHolder().findConcept().getConnectors().stream().map(Connector::getResolvedTable).collect(Collectors.toSet()); + return this.getHolder() + .findConcept() + .getConnectors() + .stream() + .map(Connector::getResolvedTable) + .collect( + Collectors.toSet()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/QuarterSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/QuarterSelect.java index b46624ef86d..4828d12ecc5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/QuarterSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/QuarterSelect.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.models.datasets.concepts.select.concept.specific; -import java.sql.ResultSet; -import java.sql.SQLException; +import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.apiv1.query.TemporalSamplerFactory; import com.bakdata.conquery.io.cps.CPSType; @@ -12,7 +11,6 @@ import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonCreator; -import jakarta.validation.constraints.NotNull; import lombok.Data; import lombok.RequiredArgsConstructor; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java index a014283044d..8011c1894aa 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java @@ -46,8 +46,8 @@ public Printer createPrinter(PrinterFactory printerFactory, PrintSettings pri return super.createPrinter(printerFactory, printSettings); } - return new FlatMappingPrinter(new OneToManyMappingPrinter(getMapping().resolve())) - .andThen(printerFactory.getListPrinter(printerFactory.getStringPrinter(printSettings), printSettings)); + return new FlatMappingPrinter(new OneToManyMappingPrinter(getMapping().resolve())).andThen( + printerFactory.getListPrinter(printerFactory.getStringPrinter(printSettings), printSettings)); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/FirstValueSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/FirstValueSelect.java index aa43ede5aeb..10a3efc07f2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/FirstValueSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/FirstValueSelect.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector; -import java.sql.ResultSet; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.common.Range; @@ -12,7 +11,6 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.FirstValueAggregator; import com.bakdata.conquery.sql.conversion.model.select.FirstValueSelectConverter; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; -import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonCreator; @CPSType(id = "FIRST", base = Select.class) @@ -21,8 +19,9 @@ public class FirstValueSelect extends MappableSingleColumnSelect { @JsonCreator public FirstValueSelect( - ColumnId column, - InternToExternMapperId mapping, Range.IntegerRange substring + ColumnId column, + InternToExternMapperId mapping, + Range.IntegerRange substring ) { super(column, mapping, substring); } @@ -33,7 +32,6 @@ public Aggregator createAggregator() { } - @Override public SelectConverter createConverter() { return new FirstValueSelectConverter(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/LastValueSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/LastValueSelect.java index e5c1fd01a43..f5c85555b13 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/LastValueSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/LastValueSelect.java @@ -17,8 +17,9 @@ public class LastValueSelect extends MappableSingleColumnSelect { @JsonCreator public LastValueSelect( - ColumnId column, - InternToExternMapperId mapping, Range.IntegerRange substring + ColumnId column, + InternToExternMapperId mapping, + Range.IntegerRange substring ) { super(column, mapping, substring); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/RandomValueSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/RandomValueSelect.java index 1bc92c9d7af..77b5cbe2aea 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/RandomValueSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/RandomValueSelect.java @@ -16,8 +16,9 @@ public class RandomValueSelect extends MappableSingleColumnSelect { @JsonCreator public RandomValueSelect( - ColumnId column, - InternToExternMapperId mapping, Range.IntegerRange substring + ColumnId column, + InternToExternMapperId mapping, + Range.IntegerRange substring ) { super(column, mapping, substring); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/SingleColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/SingleColumnSelect.java index 237e2358da1..0e01b8c505d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/SingleColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/SingleColumnSelect.java @@ -1,10 +1,10 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector; +import jakarta.validation.constraints.NotNull; import java.util.Collections; import java.util.EnumSet; import java.util.List; import java.util.Set; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.models.datasets.concepts.Connector; @@ -50,7 +50,7 @@ public EnumSet getAcceptedColumnTypes() { @Override public SelectResultInfo getResultInfo(CQConcept cqConcept) { - if(categorical){ + if (categorical) { return new SelectResultInfo(this, cqConcept, Set.of(new SemanticType.CategoricalT())); } @@ -72,7 +72,12 @@ public boolean isValidColumnType() { return true; } - log.error("Column[{}] is of Type[{}]. Not one of {} for {}", column, type, getAcceptedColumnTypes(), getClass().getSimpleName()); + log.error( + "Column[{}] is of Type[{}]. Not one of {} for {}", + column, + type, + getAcceptedColumnTypes(), + getClass().getSimpleName()); return false; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountQuartersSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountQuartersSelect.java index 0eabfac5647..8938a8e9f2f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountQuartersSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountQuartersSelect.java @@ -1,5 +1,8 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector.specific; +import java.util.List; +import javax.annotation.Nullable; + import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.DaterangeSelectOrFilter; @@ -16,9 +19,6 @@ import lombok.NoArgsConstructor; import lombok.Setter; -import javax.annotation.Nullable; -import java.util.List; - /** * Entity is included when the number of distinct quarters for all events is within a given range. * Implementation is specific for DateRanges diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountSelect.java index 6812eddb0a3..3bcbbc2c953 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountSelect.java @@ -1,9 +1,9 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector.specific; +import jakarta.validation.constraints.NotNull; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.Column; @@ -42,7 +42,9 @@ public Aggregator createAggregator() { } if (distinctByColumn != null && !getDistinctByColumn().isEmpty()) { - return new DistinctValuesWrapperAggregator(new CountAggregator(resolved), getDistinctByColumn().stream().map(ColumnId::resolve).toList()); + return new DistinctValuesWrapperAggregator( + new CountAggregator(resolved), + getDistinctByColumn().stream().map(ColumnId::resolve).toList()); } return new DistinctValuesWrapperAggregator(new CountAggregator(resolved), List.of(getColumn().resolve())); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateDistanceSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateDistanceSelect.java index 76bb5981636..7bceaf7329f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateDistanceSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateDistanceSelect.java @@ -1,10 +1,8 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector.specific; -import java.sql.ResultSet; -import java.sql.SQLException; +import jakarta.validation.constraints.NotNull; import java.time.temporal.ChronoUnit; import java.util.EnumSet; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.select.Select; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java index 177134a31d0..400ecbc6ecb 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector.specific; -import java.sql.ResultSet; -import java.sql.SQLException; import java.util.List; import javax.annotation.Nullable; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java index ff74ef6435d..917b6d6eafc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector.specific; -import java.sql.ResultSet; -import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import javax.annotation.Nullable; @@ -22,69 +20,67 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Data; -import lombok.NoArgsConstructor; @Data @CPSType(id = "DURATION_SUM", base = Select.class) @JsonIgnoreProperties("categorical") public class DurationSumSelect extends Select implements DaterangeSelectOrFilter { - @Nullable - private ColumnId column; - - @Nullable - private ColumnId startColumn, endColumn; - - private List distinctBy; - - @Override - public List getRequiredColumns() { - List out = new ArrayList<>(); - - if (column != null) { - out.add(column); - } else { - out.add(startColumn); - out.add(endColumn); - } - - if (hasDistinct()) { - out.addAll(distinctBy); - } - return out; - } - - @Override - public ResultSetProcessor.Reader createResultSetReader(ResultSetProcessor processor) { - return processor::getInteger; - } - - @JsonIgnore - private boolean hasDistinct() { - return distinctBy != null && !distinctBy.isEmpty(); - } - - @Override - public Aggregator createAggregator() { - ColumnAggregator aggregator = isSingleColumnDaterange() - ? new DurationSumAggregator(getColumn().resolve()) - : new TwoColumnDurationSumAggregator(startColumn.resolve(), endColumn.resolve()); - - if (!hasDistinct()) { - return aggregator; - } - - return new DistinctValuesWrapperAggregator<>(aggregator, distinctBy.stream().map(ColumnId::resolve).toList()); - } - - @Override - public ResultType getResultType() { - return ResultType.Primitive.INTEGER; - } - - @Override - public SelectConverter createConverter() { - //TODO apply distinctBy (though needs to be done once other branches are merged) - return new DurationSumSqlAggregator(); - } + @Nullable + private ColumnId column; + + @Nullable + private ColumnId startColumn, endColumn; + + private List distinctBy; + + @Override + public List getRequiredColumns() { + List out = new ArrayList<>(); + + if (column != null) { + out.add(column); + } else { + out.add(startColumn); + out.add(endColumn); + } + + if (hasDistinct()) { + out.addAll(distinctBy); + } + return out; + } + + @Override + public ResultSetProcessor.Reader createResultSetReader(ResultSetProcessor processor) { + return processor::getInteger; + } + + @JsonIgnore + private boolean hasDistinct() { + return distinctBy != null && !distinctBy.isEmpty(); + } + + @Override + public Aggregator createAggregator() { + ColumnAggregator aggregator = isSingleColumnDaterange() ? new DurationSumAggregator( + getColumn().resolve()) : new TwoColumnDurationSumAggregator(startColumn.resolve(), endColumn.resolve()); + + if (!hasDistinct()) { + return aggregator; + } + + return new DistinctValuesWrapperAggregator<>(aggregator, distinctBy.stream().map(ColumnId::resolve).toList()); + } + + @Override + public ResultType getResultType() { + return ResultType.Primitive.INTEGER; + } + + @Override + public SelectConverter createConverter() { + //TODO apply distinctBy (though needs to be done once other branches are merged) + return new DurationSumSqlAggregator(); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/FlagSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/FlagSelect.java index 2679f753649..00af1eb8ccb 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/FlagSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/FlagSelect.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector.specific; -import java.sql.ResultSet; -import java.sql.SQLException; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -52,7 +50,10 @@ public ResultSetProcessor.Reader> createResultSetReader(ResultSetPr @Override public Aggregator createAggregator() { - final Map collect = flags.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().resolve())); + final Map collect = flags.entrySet() + .stream() + .collect( + Collectors.toMap(Map.Entry::getKey, e -> e.getValue().resolve())); return new FlagsAggregator(collect); } @@ -66,7 +67,12 @@ public boolean isAllColumnsOfSameTable() { @JsonIgnore @ValidationMethod(message = "Columns must be BOOLEAN.") public boolean isAllColumnsBoolean() { - return flags.values().stream().map(ColumnId::resolve).map(Column::getType).allMatch(MajorTypeId.BOOLEAN::equals); + return flags.values() + .stream() + .map(ColumnId::resolve) + .map(Column::getType) + .allMatch( + MajorTypeId.BOOLEAN::equals); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java index 8900fa8d2f8..a4fde8d7e9b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java @@ -4,10 +4,10 @@ import static com.bakdata.conquery.models.types.ResultType.resolveResultType; import static org.jooq.impl.DSL.*; +import jakarta.validation.Valid; import java.util.Collections; import java.util.Set; import javax.annotation.Nullable; -import jakarta.validation.Valid; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.io.jackson.View; @@ -51,27 +51,33 @@ public abstract class MappableSingleColumnSelect extends SingleColumnSelect { private final Range.IntegerRange substringRange; - public MappableSingleColumnSelect(ColumnId column, @Nullable InternToExternMapperId mapping, @Nullable Range.IntegerRange substringRange) { + public MappableSingleColumnSelect( + ColumnId column, + @Nullable InternToExternMapperId mapping, + @Nullable Range.IntegerRange substringRange) { super(column); this.mapping = mapping; this.substringRange = substringRange; } public static SingleColumnSqlSelect getSubstringSelect( - Column column, Range.IntegerRange substringRange, SelectContext selectContext, - String alias) { + Column column, + Range.IntegerRange substringRange, + SelectContext selectContext, + String alias) { Field field = field(name(selectContext.getTables().getRootTable(), column.getName()), String.class); if (substringRange != null && !substringRange.isAll()) { if (substringRange.isAtLeast()) { field = substring(field, 1 + substringRange.getMin()); - } - else if (substringRange.isAtMost()) { + } else if (substringRange.isAtMost()) { field = substring(field, 1, substringRange.getMax()); - } - else { - field = substring(field, 1 + substringRange.getMin(), substringRange.getMax() - substringRange.getMin()); + } else { + field = substring( + field, + 1 + substringRange.getMin(), + substringRange.getMax() - substringRange.getMin()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/PrefixSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/PrefixSelect.java index d098c9d796a..c161375daf1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/PrefixSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/PrefixSelect.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector.specific; -import java.sql.ResultSet; -import java.sql.SQLException; import java.util.EnumSet; import com.bakdata.conquery.io.cps.CPSType; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/QuartersInYearSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/QuartersInYearSelect.java index 70f5385c5a1..92982e2dc88 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/QuartersInYearSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/QuartersInYearSelect.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector.specific; -import java.sql.ResultSet; -import java.sql.SQLException; import java.util.EnumSet; import com.bakdata.conquery.io.cps.CPSType; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/SumSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/SumSelect.java index c620a327a8b..d7e2d1382d7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/SumSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/SumSelect.java @@ -1,12 +1,10 @@ package com.bakdata.conquery.models.datasets.concepts.select.connector.specific; -import java.sql.ResultSet; -import java.sql.SQLException; +import jakarta.validation.constraints.NotNull; import java.util.ArrayList; import java.util.Collections; import java.util.EnumSet; import java.util.List; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.Column; @@ -61,7 +59,9 @@ public SumSelect(ColumnId column, ColumnId subtractColumn) { @Override public Aggregator createAggregator() { if (distinctByColumn != null && !distinctByColumn.isEmpty()) { - return new DistinctValuesWrapperAggregator<>(getAggregator(), getDistinctByColumn().stream().map(ColumnId::resolve).toList()); + return new DistinctValuesWrapperAggregator<>( + getAggregator(), + getDistinctByColumn().stream().map(ColumnId::resolve).toList()); } return getAggregator(); } @@ -75,13 +75,17 @@ private ColumnAggregator getAggregator() { case MONEY -> new MoneySumAggregator(resolved); case DECIMAL -> new DecimalSumAggregator(resolved); case REAL -> new RealSumAggregator(resolved); - default -> throw new IllegalStateException(String.format("Invalid column type '%s' for SUM Aggregator", resolved.getType())); + default -> throw new IllegalStateException( + String.format("Invalid column type '%s' for SUM Aggregator", resolved.getType())); }; } Column resolvedSubstract = getSubtractColumn().resolve(); if (resolved.getType() != resolvedSubstract.getType()) { - throw new IllegalStateException(String.format("Column types are not the same: Column %s\tSubstractColumn %s", resolved.getType(), resolvedSubstract - .getType())); + throw new IllegalStateException( + String.format( + "Column types are not the same: Column %s\tSubstractColumn %s", + resolved.getType(), + resolvedSubstract.getType())); } return switch (resolved.getType()) { @@ -89,12 +93,17 @@ private ColumnAggregator getAggregator() { case MONEY -> new MoneyDiffSumAggregator(resolved, resolvedSubstract); case DECIMAL -> new DecimalDiffSumAggregator(resolved, resolvedSubstract); case REAL -> new RealDiffSumAggregator(resolved, resolvedSubstract); - default -> throw new IllegalStateException(String.format("Invalid column type '%s' for SUM Aggregator", resolved.getType())); + default -> throw new IllegalStateException( + String.format("Invalid column type '%s' for SUM Aggregator", resolved.getType())); }; } - private static final EnumSet NUMBER_COMPATIBLE = EnumSet.of(MajorTypeId.INTEGER, MajorTypeId.MONEY, MajorTypeId.DECIMAL, MajorTypeId.REAL); + private static final EnumSet NUMBER_COMPATIBLE = EnumSet.of( + MajorTypeId.INTEGER, + MajorTypeId.MONEY, + MajorTypeId.DECIMAL, + MajorTypeId.REAL); @Override public List getRequiredColumns() { @@ -137,7 +146,10 @@ public boolean isSummableColumnType() { @ValidationMethod(message = "Columns are not of same Type.") @JsonIgnore public boolean isColumnsOfSameType() { - return getSubtractColumn() == null || getSubtractColumn().resolve().getType().equals(getColumn().resolve().getType()); + return getSubtractColumn() == null || getSubtractColumn().resolve() + .getType() + .equals( + getColumn().resolve().getType()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeCache.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeCache.java index 23588dc9c42..c011aa91b86 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeCache.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeCache.java @@ -41,9 +41,11 @@ public class ConceptTreeCache { /** * If id is already in cache, use that. If not calculate it by querying treeConcept. If rowMap was not used to query, cache the response. */ - public ConceptTreeChild findMostSpecificChild(String value, CalculatedValue> rowMap) throws ConceptConfigurationException { + public ConceptTreeChild findMostSpecificChild( + String value, + CalculatedValue> rowMap) throws ConceptConfigurationException { - if(cached.containsKey(value)) { + if (cached.containsKey(value)) { hits++; return cached.get(value).orElse(null); } @@ -52,7 +54,7 @@ public ConceptTreeChild findMostSpecificChild(String value, CalculatedValue parts) { for (ConceptTreeChild child : children) { if (parts.get(0).equals(child.getName())) { - final List subList = parts.size() > 1 ? parts.subList(1, parts.size()) : Collections.emptyList(); + final List subList = parts.size() > 1 ? parts.subList( + 1, + parts.size()) : Collections.emptyList(); return child.findByParts(subList); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeConnector.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeConnector.java index 6e541bd53ea..edab8bb1825 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeConnector.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeConnector.java @@ -1,9 +1,9 @@ package com.bakdata.conquery.models.datasets.concepts.tree; +import jakarta.validation.Valid; import java.util.ArrayList; import java.util.List; import javax.annotation.CheckForNull; -import jakarta.validation.Valid; import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.datasets.concepts.Connector; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/TreeConcept.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/TreeConcept.java index 16a845732f2..4fae8714391 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/TreeConcept.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/TreeConcept.java @@ -1,12 +1,12 @@ package com.bakdata.conquery.models.datasets.concepts.tree; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.stream.Stream; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.jackson.Initializing; @@ -34,7 +34,7 @@ @Slf4j @CPSType(id = "TREE", base = Concept.class) @JsonDeserialize(converter = TreeConcept.Initializer.class) -@EqualsAndHashCode(callSuper=true) +@EqualsAndHashCode(callSuper = true) public class TreeConcept extends Concept implements SelectHolder { @JsonIgnore @@ -88,12 +88,16 @@ public Stream getAllChildren() { return localIdMap.stream().filter(ConceptTreeChild.class::isInstance).map(ConceptTreeChild.class::cast); } - public ConceptTreeChild findMostSpecificChild(String stringValue, CalculatedValue> rowMap) throws ConceptConfigurationException { + public ConceptTreeChild findMostSpecificChild( + String stringValue, + CalculatedValue> rowMap) throws ConceptConfigurationException { return findMostSpecificChild(stringValue, rowMap, getChildren()); } - private ConceptTreeChild findMostSpecificChild(String stringValue, CalculatedValue> rowMap, List currentList) - throws ConceptConfigurationException { + private ConceptTreeChild findMostSpecificChild( + String stringValue, + CalculatedValue> rowMap, + List currentList) throws ConceptConfigurationException { ConceptTreeChild best = null; @@ -107,8 +111,13 @@ private ConceptTreeChild findMostSpecificChild(String stringValue, CalculatedVal if (match != null) { failed = true; - log.error("Value '{}' matches the two nodes {} and {} in the tree {} (row={}))" - , stringValue, match.getId(), n.getId(), n.getConcept().getId(), rowMap.getValue() + log.error( + "Value '{}' matches the two nodes {} and {} in the tree {} (row={}))", + stringValue, + match.getId(), + n.getId(), + n.getConcept().getId(), + rowMap.getValue() ); continue; } @@ -180,9 +189,10 @@ public void initElements() throws ConfigurationException, JSONException { ctc.setDepth(ctc.getParent().getDepth() + 1); ctc.init(); - } - catch (Exception e) { - throw new RuntimeException("Error trying to consolidate the node %s in %s".formatted(ctc.getLabel(), getLabel()), e); + } catch (Exception e) { + throw new RuntimeException( + "Error trying to consolidate the node %s in %s".formatted(ctc.getLabel(), getLabel()), + e); } openList.addAll((openList.get(i)).getChildren()); @@ -218,7 +228,9 @@ public ConceptElement> findById(ConceptElementId subParts = parts.size() > 3 ? parts.subList(3, parts.size()) : Collections.emptyList(); + final List subParts = parts.size() > 3 ? parts.subList( + 3, + parts.size()) : Collections.emptyList(); return child.findByParts(subParts); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/error/ConqueryError.java b/backend/src/main/java/com/bakdata/conquery/models/error/ConqueryError.java index c2fa62366b6..e2c411c9513 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/error/ConqueryError.java +++ b/backend/src/main/java/com/bakdata/conquery/models/error/ConqueryError.java @@ -1,8 +1,8 @@ package com.bakdata.conquery.models.error; +import jakarta.validation.constraints.NotNull; import java.util.Set; import java.util.UUID; -import jakarta.validation.constraints.NotNull; import c10n.C10N; import com.bakdata.conquery.io.cps.CPSBase; diff --git a/backend/src/main/java/com/bakdata/conquery/models/error/ConqueryErrorInfo.java b/backend/src/main/java/com/bakdata/conquery/models/error/ConqueryErrorInfo.java index 5875d532495..9b83b9d3d98 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/error/ConqueryErrorInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/error/ConqueryErrorInfo.java @@ -12,21 +12,22 @@ public interface ConqueryErrorInfo { * A unique id for this error to retrieve it in the logs. */ UUID getId(); - + String getCode(); - + String getMessage(); SimpleErrorInfo asPlain(); - + /** * Method to check if two errors are basically the same, by not checking the id and the context (which possibly checks on hashcode basis). */ default boolean equalsRegardingCodeAndMessage(ConqueryErrorInfo other) { - return Comparator - .comparing(ConqueryErrorInfo::getCode) + return Comparator.comparing(ConqueryErrorInfo::getCode) .thenComparing(ConqueryErrorInfo::getMessage) - .compare(this, other) == 0; + .compare( + this, + other) == 0; } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/error/SimpleErrorInfo.java b/backend/src/main/java/com/bakdata/conquery/models/error/SimpleErrorInfo.java index bca7602c9ec..dd0424e7498 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/error/SimpleErrorInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/error/SimpleErrorInfo.java @@ -1,8 +1,8 @@ package com.bakdata.conquery.models.error; +import jakarta.validation.constraints.NotNull; import java.util.UUID; -import jakarta.validation.constraints.NotNull; import lombok.Data; /** diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/Bucket.java b/backend/src/main/java/com/bakdata/conquery/models/events/Bucket.java index e5e54f0f8f2..25d1599a607 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/Bucket.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/Bucket.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.events; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotNull; import java.math.BigDecimal; import java.util.Arrays; import java.util.Collection; @@ -7,8 +9,6 @@ import java.util.Map; import java.util.Objects; import java.util.function.IntFunction; -import jakarta.validation.constraints.Min; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.common.daterange.CDateRange; @@ -80,15 +80,22 @@ public static Bucket fromPreprocessed(Table table, PreprocessedData container, I final ColumnStore[] storesSorted = sortColumns(table, container.getStores()); final int numberOfEvents = container.getEnds().values().stream().mapToInt(i -> i).max().orElse(0); - return new Bucket(container.getBucketId(), new Object2IntOpenHashMap<>(container.getStarts()), new Object2IntOpenHashMap<>(container.getEnds()), numberOfEvents, imp.getId(), storesSorted); + return new Bucket( + container.getBucketId(), + new Object2IntOpenHashMap<>(container.getStarts()), + new Object2IntOpenHashMap<>(container.getEnds()), + numberOfEvents, + imp.getId(), + storesSorted); } private static ColumnStore[] sortColumns(Table table, Map stores) { return Arrays.stream(table.getColumns()) - .map(Column::getName) - .map(stores::get) - .map(Objects::requireNonNull) - .toArray(ColumnStore[]::new); + .map(Column::getName) + .map(stores::get) + .map( + Objects::requireNonNull) + .toArray(ColumnStore[]::new); } @JsonIgnore @@ -162,7 +169,7 @@ public int getDate(int event, @NotNull Column column) { public boolean eventIsContainedIn(int event, ValidityDate validityDate, CDateSet dateRanges) { final CDateRange dateRange = validityDate.getValidityDate(event, this); - if (dateRange == null){ + if (dateRange == null) { return false; } @@ -185,7 +192,7 @@ public Object createScriptValue(int event, @NotNull Column column) { return getStore(column).createScriptValue(event); } - public IntFunction> mapCalculator(){ + public IntFunction> mapCalculator() { Column[] columns = getTable().resolve().getColumns(); return event -> calculateMap(event, stores, columns); @@ -219,4 +226,4 @@ public DatasetId getDataset() { public ColumnStore getStore(@NotNull String storeName) { return getStore(getTable().resolve().getColumnByName(storeName)); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/BucketManager.java b/backend/src/main/java/com/bakdata/conquery/models/events/BucketManager.java index 906a909cc84..43ff64aff80 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/BucketManager.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/BucketManager.java @@ -80,7 +80,9 @@ public static BucketManager create(Worker worker, WorkerStorage storage) { cBlockIds.forEach(cBlockId -> { if (!assignedBucketNumbers.contains(cBlockId.getBucket().getBucket())) { - log.warn("Found CBlock[{}] in Storage that does not belong to this Worker according to the Worker information.", cBlockId); + log.warn( + "Found CBlock[{}] in Storage that does not belong to this Worker according to the Worker information.", + cBlockId); } registerCblockToConnector(cBlockId, connectorCBlocks); @@ -110,16 +112,26 @@ public static BucketManager create(Worker worker, WorkerStorage storage) { return new BucketManager(worker.getJobManager(), storage, worker, connectorCBlocks, tableBuckets); } - private static void registerCblockToConnector(CBlockId cBlock, Map>> connectorCBlocks) { - connectorCBlocks.computeIfAbsent(cBlock.getConnector(), connectorId -> new Int2ObjectAVLTreeMap<>()) - .computeIfAbsent(cBlock.getBucket().getBucket(), bucketId -> new HashMap<>(3)) - .put(cBlock.getBucket(), cBlock); + private static void registerCblockToConnector( + CBlockId cBlock, + Map>> connectorCBlocks) { + connectorCBlocks.computeIfAbsent( + cBlock.getConnector(), + connectorId -> new Int2ObjectAVLTreeMap<>()) + .computeIfAbsent( + cBlock.getBucket().getBucket(), + bucketId -> new HashMap<>(3)) + .put(cBlock.getBucket(), cBlock); } - private static void assignBucketToTable(BucketId bucketId, Map>> tableBuckets) { + private static void assignBucketToTable( + BucketId bucketId, + Map>> tableBuckets) { tableBuckets.computeIfAbsent(bucketId.getImp().getTable(), id -> new Int2ObjectAVLTreeMap<>()) - .computeIfAbsent(bucketId.getBucket(), n -> new HashSet<>()) - .add(bucketId); + .computeIfAbsent( + bucketId.getBucket(), + n -> new HashSet<>()) + .add(bucketId); } public synchronized void addCalculatedCBlock(CBlock cBlock) { @@ -135,10 +147,13 @@ public void addBucket(Bucket bucket) { try (Stream> concepts = storage.getAllConcepts()) { concepts.filter(TreeConcept.class::isInstance) - .flatMap(concept -> concept.getConnectors().stream()) - .filter(connector -> connector.resolveTableId().equals(bucket.getTable())) - .filter(connector -> !hasCBlock(new CBlockId(bucket.getId(), connector.getId()))) - .forEach(connector -> job.addCBlock(bucket.getId(), (ConceptTreeConnector) connector)); + .flatMap(concept -> concept.getConnectors().stream()) + .filter( + connector -> connector.resolveTableId().equals(bucket.getTable())) + .filter( + connector -> !hasCBlock(new CBlockId(bucket.getId(), connector.getId()))) + .forEach( + connector -> job.addCBlock(bucket.getId(), (ConceptTreeConnector) connector)); } for (String entity : bucket.entities()) { @@ -166,9 +181,7 @@ public void removeTable(TableId table) { // It's possible no buckets were registered yet if (removed != null) { - removed.values().stream() - .flatMap(Set::stream) - .forEach(this::removeBucket); + removed.values().stream().flatMap(Set::stream).forEach(this::removeBucket); } storage.removeTable(table); @@ -176,11 +189,14 @@ public void removeTable(TableId table) { public void removeBucket(BucketId bucket) { try (Stream cBlockIds = storage.getAllCBlockIds()) { - cBlockIds.filter(cblock -> cblock.getBucket().equals(bucket)) - .forEach(this::removeCBlock); + cBlockIds.filter(cblock -> cblock.getBucket().equals(bucket)).forEach(this::removeCBlock); } - tableToBuckets.getOrDefault(bucket.getImp().getTable(), Int2ObjectMaps.emptyMap()).getOrDefault(bucket.getBucket(), Collections.emptySet()).remove(bucket); + tableToBuckets.getOrDefault(bucket.getImp().getTable(), Int2ObjectMaps.emptyMap()) + .getOrDefault( + bucket.getBucket(), + Collections.emptySet()) + .remove(bucket); storage.removeBucket(bucket); } @@ -188,9 +204,11 @@ public void removeBucket(BucketId bucket) { private void removeCBlock(CBlockId cBlock) { connectorToCblocks.getOrDefault(cBlock.getConnector(), Int2ObjectMaps.emptyMap()) - .getOrDefault(cBlock.getBucket().getBucket(), Collections.emptyMap()) - .values() - .remove(cBlock); + .getOrDefault( + cBlock.getBucket().getBucket(), + Collections.emptyMap()) + .values() + .remove(cBlock); storage.removeCBlock(cBlock); } @@ -212,7 +230,8 @@ public void removeImport(ImportId imp) { try (Stream> concepts = storage.getAllConcepts()) { concepts.filter(TreeConcept.class::isInstance) - .forEach(concept -> removeConceptTreeCacheByImport(concept.getId(), imp)); + .forEach( + concept -> removeConceptTreeCacheByImport(concept.getId(), imp)); } storage.removeImport(imp); @@ -232,7 +251,10 @@ public void removeConceptTreeCacheByImport(ConceptId concept, ImportId imp) { public Set getEntityBucketsForTable(Entity entity, TableId table) { final int bucketId = getBucket(entity.getId()); - return tableToBuckets.getOrDefault(table, Int2ObjectMaps.emptyMap()).getOrDefault(bucketId, Collections.emptySet()); + return tableToBuckets.getOrDefault(table, Int2ObjectMaps.emptyMap()) + .getOrDefault( + bucketId, + Collections.emptySet()); } private int getBucket(String id) { @@ -242,11 +264,14 @@ private int getBucket(String id) { /** * Collects all Entites, that have any of the concepts on the connectors in a specific time. */ - public Set getEntitiesWithConcepts(Collection> concepts, Set connectors, CDateSet restriction) { - List> resolvedConcepts = - concepts.stream() - .>map(ConceptElementId::resolve) - .toList(); + public Set getEntitiesWithConcepts( + Collection> concepts, + Set connectors, + CDateSet restriction) { + List> resolvedConcepts = concepts.stream() + .>map( + ConceptElementId::resolve) + .toList(); final long requiredBits = ConceptNode.calculateBitMask(resolvedConcepts); @@ -262,7 +287,8 @@ public Set getEntitiesWithConcepts(Collection> conce CBlock cBlock = cBlockId.resolve(); for (String entity : cBlock.getEntities()) { - if (cBlock.isConceptIncluded(entity, requiredBits) && restriction.intersects(cBlock.getEntityDateRange(entity))) { + if (cBlock.isConceptIncluded(entity, requiredBits) && restriction.intersects( + cBlock.getEntityDateRange(entity))) { out.add(entity); } } @@ -276,12 +302,17 @@ public Set getEntitiesWithConcepts(Collection> conce public Map getEntityCBlocksForConnector(Entity entity, ConnectorId connector) { final int bucketId = getBucket(entity.getId()); - return connectorToCblocks.getOrDefault(connector, Int2ObjectMaps.emptyMap()).getOrDefault(bucketId, Collections.emptyMap()); + return connectorToCblocks.getOrDefault(connector, Int2ObjectMaps.emptyMap()) + .getOrDefault( + bucketId, + Collections.emptyMap()); } public boolean hasEntityCBlocksForConnector(Entity entity, ConnectorId connector) { final int bucketId = getBucket(entity.getId()); - final Map cblocks = connectorToCblocks.getOrDefault(connector, Int2ObjectMaps.emptyMap()).getOrDefault(bucketId, Collections.emptyMap()); + final Map cblocks = connectorToCblocks.getOrDefault( + connector, + Int2ObjectMaps.emptyMap()).getOrDefault(bucketId, Collections.emptyMap()); for (CBlockId cblock : cblocks.values()) { if (cblock.resolve().containsEntity(entity.getId())) { @@ -330,10 +361,11 @@ public void addConcept(Concept concept) { for (ConceptTreeConnector connector : ((TreeConcept) concept).getConnectors()) { try (Stream allBuckets = storage.getAllBucketIds()) { - allBuckets - .filter(bucketId -> bucketId.getImp().getTable().equals(connector.resolveTableId())) - .filter(bucketId -> !hasCBlock(new CBlockId(bucketId, connector.getId()))) - .forEach(bucket -> job.addCBlock(bucket, connector)); + allBuckets.filter(bucketId -> bucketId.getImp().getTable().equals(connector.resolveTableId())) + .filter( + bucketId -> !hasCBlock(new CBlockId(bucketId, connector.getId()))) + .forEach( + bucket -> job.addCBlock(bucket, connector)); } } jobManager.addSlowJob(job); @@ -345,7 +377,9 @@ public void removeConceptTreeCacheByConcept(ConceptId concept) { public ConceptTreeCache getConceptTreeCache(TreeConcept concept, ImportId imp) { return treeCaches.computeIfAbsent(concept.getId(), (ignored) -> new ConcurrentHashMap<>()) - .computeIfAbsent(imp, (ignored) -> new ConceptTreeCache(concept)); + .computeIfAbsent( + imp, + (ignored) -> new ConceptTreeCache(concept)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/CBlock.java b/backend/src/main/java/com/bakdata/conquery/models/events/CBlock.java index 981c48e6228..4680c23aea0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/CBlock.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/CBlock.java @@ -1,17 +1,16 @@ package com.bakdata.conquery.models.events; +import jakarta.validation.constraints.NotNull; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.function.IntFunction; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.io.jackson.serializer.CBlockDeserializer; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.datasets.Column; -import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.models.datasets.concepts.ValidityDate; @@ -79,13 +78,10 @@ public class CBlock extends NamespacedIdentifiable { * @param depthEstimate estimate of depth of mostSpecificChildren */ public static long estimateMemoryBytes(long entities, long entries, double depthEstimate) { - return Math.round(entities * - ( - Integer.BYTES + Long.BYTES // includedConcepts - + Integer.BYTES // minDate - + Integer.BYTES // maxDate - ) - + entries * depthEstimate * Integer.BYTES // mostSpecificChildren (rough estimate, not resident on ManagerNode) + return Math.round(entities * (Integer.BYTES + Long.BYTES // includedConcepts + + Integer.BYTES // minDate + + Integer.BYTES // maxDate + ) + entries * depthEstimate * Integer.BYTES // mostSpecificChildren (rough estimate, not resident on ManagerNode) ); } @@ -98,7 +94,12 @@ public static CBlock createCBlock(ConceptTreeConnector connector, BucketId bucke final Map includedConcepts = calculateConceptElementPathBloomFilter(bucket, mostSpecificChildren); final Map entitySpans = calculateEntityDateIndices(bucket, connector); - final CBlock cBlock = new CBlock(bucketId, connector.getId(), includedConcepts, entitySpans, mostSpecificChildren); + final CBlock cBlock = new CBlock( + bucketId, + connector.getId(), + includedConcepts, + entitySpans, + mostSpecificChildren); return cBlock; } @@ -106,7 +107,10 @@ public static CBlock createCBlock(ConceptTreeConnector connector, BucketId bucke * Calculates the path for each event from the root of the {@link TreeConcept} to the most specific {@link ConceptTreeChild} * denoted by the individual {@link ConceptTreeChild#getPrefix()}. */ - private static int[][] calculateSpecificChildrenPaths(Bucket bucket, ConceptTreeConnector connector, BucketManager bucketManager) { + private static int[][] calculateSpecificChildrenPaths( + Bucket bucket, + ConceptTreeConnector connector, + BucketManager bucketManager) { if (connector.getColumn() == null) { return calculateSpecificChildrenPathsWithoutColumn(bucket, connector); } @@ -120,7 +124,9 @@ private static int[][] calculateSpecificChildrenPaths(Bucket bucket, ConceptTree * This is used in the evaluation of a query to quickly decide if an event is of interest by logically ANDing * the bitmask of the event with the bitmask calculated by {@link ConceptNode#calculateBitMask(Collection)} */ - private static Map calculateConceptElementPathBloomFilter(Bucket bucket, int[][] mostSpecificChildren) { + private static Map calculateConceptElementPathBloomFilter( + Bucket bucket, + int[][] mostSpecificChildren) { final Map includedConcepts = new HashMap<>(bucket.entities().size()); for (String entity : bucket.entities()) { @@ -164,7 +170,7 @@ private static Map calculateEntityDateIndices(Bucket bucket, final CDateRange range = validityDate.getValidityDate(event, bucket); - if(range == null) { + if (range == null) { continue; } @@ -216,7 +222,8 @@ private static int[][] calculateSpecificChildrenPathsWithoutColumn(Bucket bucket // Lazy evaluation of map to avoid allocations if possible. // Copy event for closure. final int _event = event; - final CalculatedValue> rowMap = new CalculatedValue<>(() -> mapCalculator.apply(_event)); + final CalculatedValue> rowMap = new CalculatedValue<>( + () -> mapCalculator.apply(_event)); if (connector.getCondition().matches(columnValue, rowMap)) { // by default initialized to the only element, the root. @@ -224,9 +231,13 @@ private static int[][] calculateSpecificChildrenPathsWithoutColumn(Bucket bucket } mostSpecificChildren[event] = Connector.NOT_CONTAINED; - } - catch (ConceptConfigurationException ex) { - log.error("Failed to evaluate event {}, row {} against connector {}", bucket.getId(), event, connector.getId(), ex); + } catch (ConceptConfigurationException ex) { + log.error( + "Failed to evaluate event {}, row {} against connector {}", + bucket.getId(), + event, + connector.getId(), + ex); } } @@ -237,7 +248,10 @@ private static int[][] calculateSpecificChildrenPathsWithoutColumn(Bucket bucket * Calculates the path for each event from the root of the {@link TreeConcept} to the most specific {@link ConceptTreeChild} * denoted by the individual {@link ConceptTreeChild#getPrefix()}. */ - private static int[][] calculateSpecificChildrenPathsWithColumn(Bucket bucket, ConceptTreeConnector connector, BucketManager bucketManager) { + private static int[][] calculateSpecificChildrenPathsWithColumn( + Bucket bucket, + ConceptTreeConnector connector, + BucketManager bucketManager) { final Column column = connector.getColumn().resolve(); @@ -263,7 +277,8 @@ private static int[][] calculateSpecificChildrenPathsWithColumn(Bucket bucket, C // Lazy evaluation of map to avoid allocations if possible. // Copy event for closure. final int _event = event; - final CalculatedValue> rowMap = new CalculatedValue<>(() -> mapCalculator.apply(_event)); + final CalculatedValue> rowMap = new CalculatedValue<>( + () -> mapCalculator.apply(_event)); if (connector.getCondition() != null && !connector.getCondition().matches(columnValue, rowMap)) { continue; @@ -278,19 +293,23 @@ private static int[][] calculateSpecificChildrenPathsWithColumn(Bucket bucket, C } mostSpecificChildren[event] = child.getPrefix(); - } - catch (ConceptConfigurationException ex) { - log.error("Failed to resolve event {}, row {} against connector {}", bucket.getId(), event, connector.getId(), ex); + } catch (ConceptConfigurationException ex) { + log.error( + "Failed to resolve event {}, row {} against connector {}", + bucket.getId(), + event, + connector.getId(), + ex); } } log.trace( - "Hits: {}, Misses: {}, Hits/Misses: {}, %Hits: {} (Up to now)", - cache.getHits(), - cache.getMisses(), - (double) cache.getHits() / cache.getMisses(), - (double) cache.getHits() / (cache.getHits() + cache.getMisses()) + "Hits: {}, Misses: {}, Hits/Misses: {}, %Hits: {} (Up to now)", + cache.getHits(), + cache.getMisses(), + (double) cache.getHits() / cache.getMisses(), + (double) cache.getHits() / (cache.getHits() + cache.getMisses()) ); @@ -375,4 +394,4 @@ public DatasetId getDataset() { return bucket.getDataset(); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/EmptyStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/EmptyStore.java index aa32018ac3d..1cd402294ce 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/EmptyStore.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/EmptyStore.java @@ -19,8 +19,7 @@ * @implNote this class is a singleton. */ @CPSType(base = ColumnStore.class, id = "EMPTY") -public enum EmptyStore implements - IntegerStore, RealStore, BooleanStore, DecimalStore, StringStore, MoneyStore, DateStore { +public enum EmptyStore implements IntegerStore, RealStore, BooleanStore, DecimalStore, StringStore, MoneyStore, DateStore { INSTANCE; @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/MajorTypeId.java b/backend/src/main/java/com/bakdata/conquery/models/events/MajorTypeId.java index aa2a4010cf5..08804444fdd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/MajorTypeId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/MajorTypeId.java @@ -12,14 +12,8 @@ @RequiredArgsConstructor public enum MajorTypeId { - STRING(StringParser::new), - INTEGER(IntegerParser::new), - BOOLEAN(BooleanParser::new), - REAL(RealParser::new), - DECIMAL(DecimalParser::new), - MONEY(MoneyParser::new), - DATE(DateParser::new), - DATE_RANGE(DateRangeParser::new); + STRING(StringParser::new), INTEGER(IntegerParser::new), BOOLEAN(BooleanParser::new), REAL(RealParser::new), DECIMAL( + DecimalParser::new), MONEY(MoneyParser::new), DATE(DateParser::new), DATE_RANGE(DateRangeParser::new); public static final Set DATE_COMPATIBLE = EnumSet.of(DATE, DATE_RANGE); public static final Set NUMERIC = EnumSet.of(INTEGER, REAL, DECIMAL, MONEY); diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/StringStoreString.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/StringStoreString.java index d78e25e11f7..706bf465fd5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/StringStoreString.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/StringStoreString.java @@ -28,7 +28,7 @@ public static StringStoreString create(int size) { @JsonCreator public static StringStoreString withInternedStrings(String[] values) { - if(shouldIntern()) { + if (shouldIntern()) { for (int index = 0; index < values.length; index++) { values[index] = values[index] != null ? values[index].intern() : null; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/BooleanStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/BooleanStore.java index f43d1658b1b..9bbff780139 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/BooleanStore.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/BooleanStore.java @@ -12,10 +12,11 @@ public interface BooleanStore extends ColumnStore { boolean getBoolean(int event); + void setBoolean(int event, boolean value); @Override - default void setNull(int event){ + default void setNull(int event) { throw new IllegalStateException("BooleanStore cannot have Null values."); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/DateRangeStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/DateRangeStore.java index 0ce158ae841..62db1198019 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/DateRangeStore.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/DateRangeStore.java @@ -5,13 +5,13 @@ /** * {@link ColumnStore} for {@link CDateRange}. - * * See also {@link MajorTypeId#DATE_RANGE} and {@link com.bakdata.conquery.models.preproc.parser.specific.DateRangeParser}. */ public interface DateRangeStore extends ColumnStore { CDateRange getDateRange(int event); + void setDateRange(int event, CDateRange value); @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/DateStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/DateStore.java index 5dc73552887..e14e9215e8e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/DateStore.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/DateStore.java @@ -13,6 +13,7 @@ public interface DateStore extends ColumnStore { int getDate(int event); + void setDate(int event, int value); @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/DecimalStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/DecimalStore.java index 9c51d6afec6..cadb96aedb1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/DecimalStore.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/DecimalStore.java @@ -6,12 +6,12 @@ /** * {@link ColumnStore} for {@link BigDecimal} values. - * * See also {@link MajorTypeId#DECIMAL} and {@link com.bakdata.conquery.models.preproc.parser.specific.DecimalParser}. */ public interface DecimalStore extends ColumnStore { BigDecimal getDecimal(int event); + void setDecimal(int event, BigDecimal value); @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/MoneyStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/MoneyStore.java index c520e21e5ec..416112b78c7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/MoneyStore.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/MoneyStore.java @@ -12,6 +12,7 @@ public interface MoneyStore extends ColumnStore { BigDecimal getMoney(int event); + void setMoney(int event, BigDecimal money); @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/RealStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/RealStore.java index 8025c51141a..00d257be48d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/RealStore.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/RealStore.java @@ -4,12 +4,12 @@ /** * {@link ColumnStore} for real values (double/float). - * * See also {@link MajorTypeId#REAL} and {@link com.bakdata.conquery.models.preproc.parser.specific.RealParser}. */ public interface RealStore extends ColumnStore { double getReal(int event); + void setReal(int event, double value); @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/CompoundDateRangeStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/CompoundDateRangeStore.java index c116f9142b1..d006834cd43 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/CompoundDateRangeStore.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/CompoundDateRangeStore.java @@ -1,5 +1,8 @@ package com.bakdata.conquery.models.events.stores.specific; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; + import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.events.Bucket; @@ -9,8 +12,6 @@ import com.bakdata.conquery.models.events.stores.root.DateStore; import com.fasterxml.jackson.annotation.JsonBackReference; import com.fasterxml.jackson.annotation.JsonIgnore; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; import lombok.AccessLevel; import lombok.Getter; import lombok.NonNull; @@ -159,4 +160,4 @@ public boolean has(int event) { return has.getBoolean(event); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/DirectDateRangeStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/DirectDateRangeStore.java index dcef9bc254a..f8121222ccf 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/DirectDateRangeStore.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/DirectDateRangeStore.java @@ -59,15 +59,13 @@ public DirectDateRangeStore select(int[] starts, int[] length) { public void setDateRange(int event, CDateRange raw) { if (raw.hasLowerBound()) { minStore.setDate(event, raw.getMinValue()); - } - else { + } else { minStore.setNull(event); } if (raw.hasUpperBound()) { maxStore.setDate(event, raw.getMaxValue()); - } - else { + } else { maxStore.setNull(event); } } @@ -98,4 +96,4 @@ public CDateRange getDateRange(int event) { public boolean has(int event) { return minStore.has(event) || maxStore.has(event); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java index 1784c7f7c55..01459784737 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java @@ -40,7 +40,7 @@ public class MoneyIntStore implements MoneyStore, Initializing { private int decimalShift = Integer.MIN_VALUE; - public MoneyIntStore(IntegerStore store, int decimalShift){ + public MoneyIntStore(IntegerStore store, int decimalShift) { this(); this.numberType = store; this.decimalShift = decimalShift; @@ -92,13 +92,14 @@ public void setParent(Bucket bucket) { @Override public void init() { - if (decimalShift != Integer.MIN_VALUE){ + if (decimalShift != Integer.MIN_VALUE) { return; } decimalShift = config.getFrontend().getCurrency().getDecimalScale(); } - public static class MoneyIntStoreInitializer extends Initializing.Converter {} + public static class MoneyIntStoreInitializer extends Initializing.Converter { + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/QuarterDateRangeStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/QuarterDateRangeStore.java index d441f771fdf..2173ea34bbd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/QuarterDateRangeStore.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/QuarterDateRangeStore.java @@ -58,8 +58,7 @@ public void setNull(int event) { public void setDateRange(int event, CDateRange raw) { if (raw.hasLowerBound()) { store.setInteger(event, raw.getMinValue()); - } - else { + } else { throw new IllegalArgumentException("Cannot store open dates in QuarterStore"); } } @@ -80,4 +79,4 @@ public CDateRange getDateRange(int event) { public void setParent(Bucket bucket) { // not used } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/RebasingIntegerStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/RebasingIntegerStore.java index ab26940bb8d..ce4ff6c2657 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/RebasingIntegerStore.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/RebasingIntegerStore.java @@ -1,10 +1,11 @@ package com.bakdata.conquery.models.events.stores.specific; +import jakarta.validation.constraints.NotNull; + import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.events.stores.root.ColumnStore; import com.bakdata.conquery.models.events.stores.root.IntegerStore; -import jakarta.validation.constraints.NotNull; import lombok.Getter; import lombok.ToString; import org.jetbrains.annotations.Nullable; diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/ScaledDecimalStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/ScaledDecimalStore.java index 802d1d6b929..3727e62d6c8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/ScaledDecimalStore.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/ScaledDecimalStore.java @@ -80,4 +80,4 @@ public boolean has(int event) { public void setParent(Bucket bucket) { // not used } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/exceptions/ConceptConfigurationException.java b/backend/src/main/java/com/bakdata/conquery/models/exceptions/ConceptConfigurationException.java index 0128a8625b7..ef2ebe916be 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/exceptions/ConceptConfigurationException.java +++ b/backend/src/main/java/com/bakdata/conquery/models/exceptions/ConceptConfigurationException.java @@ -13,11 +13,11 @@ public class ConceptConfigurationException extends JSONException { private static final long serialVersionUID = 1L; public ConceptConfigurationException(String node, String message, Throwable cause) { - super("In Node "+node+": "+message, cause); + super("In Node " + node + ": " + message, cause); } public ConceptConfigurationException(Identifiable node, String message, Throwable cause) { - this(node.toString()+"("+node+")", message, cause); + this(node.toString() + "(" + node + ")", message, cause); } public ConceptConfigurationException(String node, String message) { @@ -25,15 +25,15 @@ public ConceptConfigurationException(String node, String message) { } public ConceptConfigurationException(Identifiable node, String message) { - this(node.toString()+"("+node+")", message, null); + this(node.toString() + "(" + node + ")", message, null); } public ConceptConfigurationException(File f, String message, Throwable e) { - super("In Concept Tree file "+LogUtil.printPath(f)+": "+message,e); + super("In Concept Tree file " + LogUtil.printPath(f) + ": " + message, e); } - + public ConceptConfigurationException(File f, String message) { - super("In Concept Tree file "+LogUtil.printPath(f)+": "+message); + super("In Concept Tree file " + LogUtil.printPath(f) + ": " + message); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/exceptions/ParsingException.java b/backend/src/main/java/com/bakdata/conquery/models/exceptions/ParsingException.java index c6c2ee29a06..a92f3a97660 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/exceptions/ParsingException.java +++ b/backend/src/main/java/com/bakdata/conquery/models/exceptions/ParsingException.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.models.exceptions; -import java.util.Objects; /** * This exception if thrown if any value could not be parsed as intended. @@ -18,10 +17,10 @@ public ParsingException(String message) { } public static ParsingException of(String value, String format) { - return new ParsingException("Failed to parse '"+Objects.toString(value)+"' as "+format); + return new ParsingException("Failed to parse '" + value + "' as " + format); } - + public static ParsingException of(String value, String format, Throwable cause) { - return new ParsingException("Failed to parse '"+Objects.toString(value)+"' as "+format, cause); + return new ParsingException("Failed to parse '" + value + "' as " + format, cause); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/exceptions/ValidatorHelper.java b/backend/src/main/java/com/bakdata/conquery/models/exceptions/ValidatorHelper.java index 269a367faec..46f8b61234e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/exceptions/ValidatorHelper.java +++ b/backend/src/main/java/com/bakdata/conquery/models/exceptions/ValidatorHelper.java @@ -1,13 +1,13 @@ package com.bakdata.conquery.models.exceptions; +import jakarta.validation.ConstraintViolation; +import jakarta.validation.ValidationException; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import jakarta.validation.ConstraintViolation; -import jakarta.validation.ValidationException; import lombok.experimental.UtilityClass; import org.slf4j.Logger; @@ -24,11 +24,13 @@ public static void failOnError(Logger log, Set> } } - public static Optional createViolationsString(Set> violations, boolean detailed) { + public static Optional createViolationsString( + Set> violations, + boolean detailed) { - Map, List>> mapByRoot = - violations.stream() - .collect(Collectors.groupingBy(v -> Optional.of(v.getRootBean()))); + Map, List>> mapByRoot = violations.stream() + .collect( + Collectors.groupingBy(v -> Optional.of(v.getRootBean()))); // Wrap grouper in Optional to also catch null values. // Combine all leaf fail reports into a single exception. @@ -47,11 +49,18 @@ public static Optional createViolationsString(Set createViolationString(objectToViolation.getKey(), objectToViolation.getValue(), detailed)) - .forEach(sb::append); + entry.getValue() + .stream() + .collect( + Collectors.groupingBy(ConstraintViolation::getLeafBean)) + .entrySet() + .stream() + .map( + objectToViolation -> createViolationString( + objectToViolation.getKey(), + objectToViolation.getValue(), + detailed)) + .forEach(sb::append); } return Optional.of(sb.toString()); } @@ -59,13 +68,15 @@ public static Optional createViolationsString(Set> String createViolationString(Object node, List violations, boolean detailed) { + private static > String createViolationString( + Object node, + List violations, + boolean detailed) { StringBuilder sb = new StringBuilder(); if (node == null) { // If validations are not directly mappable to an object, directly return the violation messages. - return violations.stream() - .map(ConstraintViolation::getMessage).collect(Collectors.joining("\n\t")); + return violations.stream().map(ConstraintViolation::getMessage).collect(Collectors.joining("\n\t")); } // Build report for a specific failing leaf. @@ -73,7 +84,7 @@ private static > String createViolationString(O sb.append("\n\tFor the leaf type '").append(node.getClass()).append("':"); } for (V violation : violations) { - // List all the violations for the specific leaf. + // List all the violations for the specific leaf. sb.append("\n\t\t- ").append(violation.getPropertyPath()).append(": ").append(violation.getMessage()); } return sb.toString(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/execution/Labelable.java b/backend/src/main/java/com/bakdata/conquery/models/execution/Labelable.java index 5ac694aad29..42b6ce7b58f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/execution/Labelable.java +++ b/backend/src/main/java/com/bakdata/conquery/models/execution/Labelable.java @@ -10,13 +10,14 @@ */ public interface Labelable { String getLabel(); + void setLabel(String label); - + default Consumer labeler() { return (patch) -> { if (patch != null && patch.getLabel() != null) { setLabel(patch.getLabel()); } - }; + }; } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java index 56c7482a58e..779b7bddaef 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.execution; +import jakarta.validation.constraints.NotNull; import java.time.Duration; import java.time.LocalDateTime; import java.time.ZoneId; @@ -10,7 +11,6 @@ import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.apiv1.execution.ExecutionStatus; import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; @@ -145,7 +145,12 @@ public abstract class ManagedExecution extends MetaIdentifiable datasetRegistry; - public ManagedExecution(@NonNull UserId owner, @NonNull DatasetId dataset, MetaStorage metaStorage, DatasetRegistry datasetRegistry, ConqueryConfig config) { + public ManagedExecution( + @NonNull UserId owner, + @NonNull DatasetId dataset, + MetaStorage metaStorage, + DatasetRegistry datasetRegistry, + ConqueryConfig config) { this.owner = owner; this.dataset = dataset; this.config = config; @@ -158,13 +163,15 @@ private static boolean canSubjectExpand(Subject subject, QueryDescription query) query.visit(namespacesIdCollector); final Set> concepts = namespacesIdCollector.getIdentifiables() - .stream() - .filter(ConceptElement.class::isInstance) - .map(ConceptElement.class::cast) - .>map(ConceptElement::getConcept) - .collect(Collectors.toSet()); - - return subject.isPermittedAll(concepts, Ability.READ); + .stream() + .filter( + ConceptElement.class::isInstance) + .map(ConceptElement.class::cast) + .>map( + ConceptElement::getConcept) + .collect(Collectors.toSet()); + + return subject.isPermittedAll(concepts, Ability.READ); } /** @@ -203,13 +210,11 @@ public Namespace getNamespace() { protected abstract void doInitExecutable(); private static boolean containsDates(QueryDescription query) { - return Visitable.stream(query) - .anyMatch(visitable -> - switch (visitable) { - case CQConcept cqConcept -> !cqConcept.isExcludeFromTimeAggregation(); - case CQExternal external -> external.containsDates(); - default -> false; - }); + return Visitable.stream(query).anyMatch(visitable -> switch (visitable) { + case CQConcept cqConcept -> !cqConcept.isExcludeFromTimeAggregation(); + case CQExternal external -> external.containsDates(); + default -> false; + }); } /** @@ -235,9 +240,12 @@ public ManagedExecutionId createId() { public void fail(ConqueryErrorInfo error) { if (this.error != null && !this.error.equalsRegardingCodeAndMessage(error)) { // Warn only again if the error is different (failed might be called per collected shard result) - log.warn("The execution [{}] failed again with:\n\t{}\n\tThe previous error was: {}", getId(), this.error, error); - } - else { + log.warn( + "The execution [{}] failed again with:\n\t{}\n\tThe previous error was: {}", + getId(), + this.error, + error); + } else { this.error = error; // Log the error, so its id is at least once in the logs log.warn("The execution [{}] failed with:\n\t{}", getId(), getError()); @@ -279,7 +287,8 @@ public void start() { Preconditions.checkArgument(isInitialized(), "The execution must have been initialized first"); if (getExecutionManager().isInfoPresent(getId())) { - Preconditions.checkArgument(getExecutionManager().getExecutionInfo(getId()).getExecutionState() != ExecutionState.RUNNING); + Preconditions.checkArgument( + getExecutionManager().getExecutionInfo(getId()).getExecutionState() != ExecutionState.RUNNING); } startTime = LocalDateTime.now(); @@ -306,7 +315,8 @@ public void setStatusBase(@NonNull Subject subject, @NonNull ExecutionStatus sta status.setShared(shared); status.setOwn(subject.isOwner(this)); status.setCreatedAt(getCreationTime().atZone(ZoneId.systemDefault())); - status.setRequiredTime((startTime != null && finishTime != null) ? ChronoUnit.MILLIS.between(startTime, finishTime) : null); + status.setRequiredTime( + (startTime != null && finishTime != null) ? ChronoUnit.MILLIS.between(startTime, finishTime) : null); status.setStartTime(startTime); status.setFinishTime(finishTime); status.setStatus(getState()); @@ -403,7 +413,10 @@ protected void setAdditionalStatusFields(Subject subject, FullExecutionStatus st /** * Sets additional fields of an {@link ExecutionStatus} when a more specific status is requested. */ - protected void setAdditionalFieldsForStatusWithSource(Subject subject, FullExecutionStatus status, Namespace namespace) { + protected void setAdditionalFieldsForStatusWithSource( + Subject subject, + FullExecutionStatus status, + Namespace namespace) { QueryDescription query = getSubmitted(); status.setCanExpand(canSubjectExpand(subject, query)); diff --git a/backend/src/main/java/com/bakdata/conquery/models/execution/Owned.java b/backend/src/main/java/com/bakdata/conquery/models/execution/Owned.java index ee76301a50b..6af13dfe6ef 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/execution/Owned.java +++ b/backend/src/main/java/com/bakdata/conquery/models/execution/Owned.java @@ -4,5 +4,5 @@ import com.bakdata.conquery.models.identifiable.ids.specific.UserId; public interface Owned extends Authorized { - UserId getOwner(); + UserId getOwner(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/execution/Shareable.java b/backend/src/main/java/com/bakdata/conquery/models/execution/Shareable.java index d1409cfaf9d..66e8ebb4ee9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/execution/Shareable.java +++ b/backend/src/main/java/com/bakdata/conquery/models/execution/Shareable.java @@ -1,78 +1,89 @@ -package com.bakdata.conquery.models.execution; - -import java.util.List; -import java.util.Set; -import java.util.function.Consumer; -import java.util.stream.Collectors; - -import com.bakdata.conquery.apiv1.MetaDataPatch; -import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.models.auth.entities.Group; -import com.bakdata.conquery.models.auth.entities.Subject; -import com.bakdata.conquery.models.auth.permissions.AbilitySets; -import com.bakdata.conquery.models.auth.permissions.Authorized; -import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; -import com.bakdata.conquery.models.identifiable.Identifiable; -import com.bakdata.conquery.models.identifiable.ids.Id; -import com.bakdata.conquery.models.identifiable.ids.specific.GroupId; -import com.bakdata.conquery.util.QueryUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Interface for classes that are able to be patched with an {@link MetaDataPatch}. - * Allows sharing of implementations among groups of a given user. - */ -public interface Shareable extends Authorized { - Logger log = LoggerFactory.getLogger(Shareable.class); - - /** - * Sets the flag that indicated if an instance is shared among groups. - */ - void setShared(boolean shared); - - default , S extends Identifiable & Shareable & Authorized> Consumer sharer( - MetaStorage storage, - Subject subject) { - if (!(this instanceof Identifiable)) { - log.warn("Cannot share {} ({}) because it does not implement Identifiable", getClass(), this); - return QueryUtils.getNoOpEntryPoint(); - } - - return (patch) -> { - if (patch == null || patch.getGroups() == null) { - return; - } - - final S shareable = (S) this; - // Collect groups that do not have access to this instance and remove their probable permission - for (Group group : storage.getAllGroups().toList()) { - if (patch.getGroups().contains(group.getId())) { - continue; - } - - log.trace("User {} unshares instance {} ({}) from owner {}.", subject, shareable.getClass().getSimpleName(), shareable.getId(), group); - - group.removePermission(shareable.createPermission(AbilitySets.SHAREHOLDER)); - } - - - // Resolve the provided groups - final Set groups = patch.getGroups().stream().map(storage::getGroup).collect(Collectors.toSet()); - - for(Group group : groups) { - final ConqueryPermission sharePermission = shareable.createPermission(AbilitySets.SHAREHOLDER); - group.addPermission(sharePermission); - - log.trace("User {} shares instance {} ({}). Adding permission {} to owner {}.", subject, shareable.getClass().getSimpleName(), shareable.getId(), sharePermission, group); - } - - setShared(!patch.getGroups().isEmpty()); - }; - - } - - interface ShareInformation { - List getGroups(); - } -} +package com.bakdata.conquery.models.execution; + +import java.util.List; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import com.bakdata.conquery.apiv1.MetaDataPatch; +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.models.auth.entities.Group; +import com.bakdata.conquery.models.auth.entities.Subject; +import com.bakdata.conquery.models.auth.permissions.AbilitySets; +import com.bakdata.conquery.models.auth.permissions.Authorized; +import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; +import com.bakdata.conquery.models.identifiable.Identifiable; +import com.bakdata.conquery.models.identifiable.ids.Id; +import com.bakdata.conquery.models.identifiable.ids.specific.GroupId; +import com.bakdata.conquery.util.QueryUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Interface for classes that are able to be patched with an {@link MetaDataPatch}. + * Allows sharing of implementations among groups of a given user. + */ +public interface Shareable extends Authorized { + Logger log = LoggerFactory.getLogger(Shareable.class); + + /** + * Sets the flag that indicated if an instance is shared among groups. + */ + void setShared(boolean shared); + + default , S extends Identifiable & Shareable & Authorized> Consumer sharer( + MetaStorage storage, + Subject subject) { + if (!(this instanceof Identifiable)) { + log.warn("Cannot share {} ({}) because it does not implement Identifiable", getClass(), this); + return QueryUtils.getNoOpEntryPoint(); + } + + return (patch) -> { + if (patch == null || patch.getGroups() == null) { + return; + } + + final S shareable = (S) this; + // Collect groups that do not have access to this instance and remove their probable permission + for (Group group : storage.getAllGroups().toList()) { + if (patch.getGroups().contains(group.getId())) { + continue; + } + + log.trace( + "User {} unshares instance {} ({}) from owner {}.", + subject, + shareable.getClass().getSimpleName(), + shareable.getId(), + group); + + group.removePermission(shareable.createPermission(AbilitySets.SHAREHOLDER)); + } + + + // Resolve the provided groups + final Set groups = patch.getGroups().stream().map(storage::getGroup).collect(Collectors.toSet()); + + for (Group group : groups) { + final ConqueryPermission sharePermission = shareable.createPermission(AbilitySets.SHAREHOLDER); + group.addPermission(sharePermission); + + log.trace( + "User {} shares instance {} ({}). Adding permission {} to owner {}.", + subject, + shareable.getClass().getSimpleName(), + shareable.getId(), + sharePermission, + group); + } + + setShared(!patch.getGroups().isEmpty()); + }; + + } + + interface ShareInformation { + List getGroups(); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/execution/Taggable.java b/backend/src/main/java/com/bakdata/conquery/models/execution/Taggable.java index 41185a75a18..ca1d3445f5d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/execution/Taggable.java +++ b/backend/src/main/java/com/bakdata/conquery/models/execution/Taggable.java @@ -11,6 +11,7 @@ */ public interface Taggable { String[] getTags(); + void setTags(String[] tags); default Consumer tagger() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/configs/FormConfig.java b/backend/src/main/java/com/bakdata/conquery/models/forms/configs/FormConfig.java index bdc7b2fa229..06720d7bddc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/configs/FormConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/configs/FormConfig.java @@ -1,211 +1,216 @@ -package com.bakdata.conquery.models.forms.configs; - -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Set; -import java.util.UUID; -import java.util.function.Consumer; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; - -import com.bakdata.conquery.apiv1.FormConfigPatch; -import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.models.auth.entities.Group; -import com.bakdata.conquery.models.auth.entities.Subject; -import com.bakdata.conquery.models.auth.entities.User; -import com.bakdata.conquery.models.auth.permissions.Ability; -import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; -import com.bakdata.conquery.models.auth.permissions.FormConfigPermission; -import com.bakdata.conquery.models.execution.Labelable; -import com.bakdata.conquery.models.execution.Owned; -import com.bakdata.conquery.models.execution.Shareable; -import com.bakdata.conquery.models.execution.Taggable; -import com.bakdata.conquery.models.identifiable.MetaIdentifiable; -import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; -import com.bakdata.conquery.models.identifiable.ids.specific.FormConfigId; -import com.bakdata.conquery.models.identifiable.ids.specific.GroupId; -import com.bakdata.conquery.models.identifiable.ids.specific.UserId; -import com.bakdata.conquery.util.VariableDefaultValue; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.databind.JsonNode; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.NonNull; -import lombok.ToString; -import lombok.experimental.FieldNameConstants; -import lombok.experimental.SuperBuilder; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.ArrayUtils; -import org.apache.shiro.authz.Permission; -import org.jetbrains.annotations.Nullable; - -@Slf4j -@Data -@AllArgsConstructor -@NoArgsConstructor -@ToString -@EqualsAndHashCode(callSuper = false) -@FieldNameConstants -public class FormConfig extends MetaIdentifiable implements Shareable, Labelable, Taggable, Owned { - - protected DatasetId dataset; - @NotEmpty - private String formType; - @VariableDefaultValue @NonNull - private UUID formId = UUID.randomUUID(); - private String label; - @NotNull - private String[] tags = ArrayUtils.EMPTY_STRING_ARRAY; - private boolean shared = false; - - /** - * This is a blackbox for us at the moment, where the front end saves the state of the - * form, when the user saved it. - */ - @NotNull - private JsonNode values; - private UserId owner; - @VariableDefaultValue - private LocalDateTime creationTime = LocalDateTime.now(); - - - public FormConfig(String formType, JsonNode values) { - this.formType = formType; - this.values = values; - } - - @Override - public FormConfigId createId() { - return new FormConfigId(dataset, formType, formId); - } - - /** - * Provides an overview (meta data) of this form configuration without the - * actual form field values. - */ - public FormConfigOverviewRepresentation overview(Subject subject) { - String ownerName = getOwnerName(); - - return FormConfigOverviewRepresentation.builder() - .id(getId()) - .formType(formType) - .label(label) - .tags(tags) - .ownerName(ownerName) - .own(subject.isOwner(this)) - .createdAt(getCreationTime().atZone(ZoneId.systemDefault())) - .shared(shared) - // system? - .build(); - } - - @JsonIgnore - @Nullable - private String getOwnerName() { - if (owner == null){ - return null; - } - - User resolved = owner.get(); - - if (resolved == null){ - return null; - } - - return resolved.getLabel(); - } - - /** - * Return the full representation of the configuration with the configured form fields and meta data. - */ - public FormConfigFullRepresentation fullRepresentation(MetaStorage storage, Subject requestingUser){ - String ownerName = getOwnerName(); - - /* Calculate which groups can see this query. - * This is usually not done very often and should be reasonable fast, so don't cache this. - */ - - List permittedGroups = new ArrayList<>(); - for (Group group : storage.getAllGroups().toList()) { - for(Permission perm : group.getPermissions()) { - if(perm.implies(createPermission(Ability.READ.asSet()))) { - permittedGroups.add(group.getId()); - } - } - } - - return FormConfigFullRepresentation.builder() - .id(getId()) - .formType(formType) - .label(label) - .tags(tags) - .ownerName(ownerName) - .own(requestingUser.isOwner(this)) - .createdAt(getCreationTime().atZone(ZoneId.systemDefault())) - .shared(shared) - .groups(permittedGroups) - // system? TODO discuss how system is determined (may check if owning user is in a special system group or so) - .values(values) - .build(); - } - - @Override - public ConqueryPermission createPermission(Set abilities) { - return FormConfigPermission.onInstance(abilities, getId()); - } - - public Consumer valueSetter() { - return (patch) -> setValues(patch.getValues()); - } - - /** - * API representation for the overview of all {@link FormConfig}s which does not - * include the form fields an their values. - */ - @Getter - @SuperBuilder - @ToString - @EqualsAndHashCode(callSuper = false) - @FieldNameConstants - public static class FormConfigOverviewRepresentation { - - private FormConfigId id; - private String formType; - private String label; - private String[] tags; - - private String ownerName; - private ZonedDateTime createdAt; - private boolean own; - private boolean shared; - private boolean system; - - } - - /** - * API representation for a single {@link FormConfig} which includes the form - * fields an their values. - */ - @Getter - @SuperBuilder - @ToString(callSuper = true) - @EqualsAndHashCode(callSuper = true) - @FieldNameConstants - public static class FormConfigFullRepresentation extends FormConfigOverviewRepresentation { - - /** - * The groups this config is shared with. - */ - private Collection groups; - - private JsonNode values; - } - -} +package com.bakdata.conquery.models.forms.configs; + +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.function.Consumer; + +import com.bakdata.conquery.apiv1.FormConfigPatch; +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.models.auth.entities.Group; +import com.bakdata.conquery.models.auth.entities.Subject; +import com.bakdata.conquery.models.auth.entities.User; +import com.bakdata.conquery.models.auth.permissions.Ability; +import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; +import com.bakdata.conquery.models.auth.permissions.FormConfigPermission; +import com.bakdata.conquery.models.execution.Labelable; +import com.bakdata.conquery.models.execution.Owned; +import com.bakdata.conquery.models.execution.Shareable; +import com.bakdata.conquery.models.execution.Taggable; +import com.bakdata.conquery.models.identifiable.MetaIdentifiable; +import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; +import com.bakdata.conquery.models.identifiable.ids.specific.FormConfigId; +import com.bakdata.conquery.models.identifiable.ids.specific.GroupId; +import com.bakdata.conquery.models.identifiable.ids.specific.UserId; +import com.bakdata.conquery.util.VariableDefaultValue; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.JsonNode; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import lombok.ToString; +import lombok.experimental.FieldNameConstants; +import lombok.experimental.SuperBuilder; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.shiro.authz.Permission; +import org.jetbrains.annotations.Nullable; + +@Slf4j +@Data +@AllArgsConstructor +@NoArgsConstructor +@ToString +@EqualsAndHashCode(callSuper = false) +@FieldNameConstants +public class FormConfig extends MetaIdentifiable implements Shareable, Labelable, Taggable, Owned { + + protected DatasetId dataset; + @NotEmpty + private String formType; + @VariableDefaultValue + @NonNull + private UUID formId = UUID.randomUUID(); + private String label; + @NotNull + private String[] tags = ArrayUtils.EMPTY_STRING_ARRAY; + private boolean shared = false; + + /** + * This is a blackbox for us at the moment, where the front end saves the state of the + * form, when the user saved it. + */ + @NotNull + private JsonNode values; + private UserId owner; + @VariableDefaultValue + private LocalDateTime creationTime = LocalDateTime.now(); + + + public FormConfig(String formType, JsonNode values) { + this.formType = formType; + this.values = values; + } + + @Override + public FormConfigId createId() { + return new FormConfigId(dataset, formType, formId); + } + + /** + * Provides an overview (meta data) of this form configuration without the + * actual form field values. + */ + public FormConfigOverviewRepresentation overview(Subject subject) { + String ownerName = getOwnerName(); + + return FormConfigOverviewRepresentation.builder() + .id(getId()) + .formType(formType) + .label(label) + .tags( + tags) + .ownerName(ownerName) + .own(subject.isOwner(this)) + .createdAt( + getCreationTime().atZone(ZoneId.systemDefault())) + .shared(shared) + // system? + .build(); + } + + @JsonIgnore + @Nullable + private String getOwnerName() { + if (owner == null) { + return null; + } + + User resolved = owner.get(); + + if (resolved == null) { + return null; + } + + return resolved.getLabel(); + } + + /** + * Return the full representation of the configuration with the configured form fields and meta data. + */ + public FormConfigFullRepresentation fullRepresentation(MetaStorage storage, Subject requestingUser) { + String ownerName = getOwnerName(); + + /* Calculate which groups can see this query. + * This is usually not done very often and should be reasonable fast, so don't cache this. + */ + + List permittedGroups = new ArrayList<>(); + for (Group group : storage.getAllGroups().toList()) { + for (Permission perm : group.getPermissions()) { + if (perm.implies(createPermission(Ability.READ.asSet()))) { + permittedGroups.add(group.getId()); + } + } + } + + return FormConfigFullRepresentation.builder() + .id(getId()) + .formType(formType) + .label(label) + .tags(tags) + .ownerName( + ownerName) + .own(requestingUser.isOwner(this)) + .createdAt( + getCreationTime().atZone(ZoneId.systemDefault())) + .shared(shared) + .groups(permittedGroups) + // system? TODO discuss how system is determined (may check if owning user is in a special system group or so) + .values(values) + .build(); + } + + @Override + public ConqueryPermission createPermission(Set abilities) { + return FormConfigPermission.onInstance(abilities, getId()); + } + + public Consumer valueSetter() { + return (patch) -> setValues(patch.getValues()); + } + + /** + * API representation for the overview of all {@link FormConfig}s which does not + * include the form fields an their values. + */ + @Getter + @SuperBuilder + @ToString + @EqualsAndHashCode(callSuper = false) + @FieldNameConstants + public static class FormConfigOverviewRepresentation { + + private FormConfigId id; + private String formType; + private String label; + private String[] tags; + + private String ownerName; + private ZonedDateTime createdAt; + private boolean own; + private boolean shared; + private boolean system; + + } + + /** + * API representation for a single {@link FormConfig} which includes the form + * fields an their values. + */ + @Getter + @SuperBuilder + @ToString(callSuper = true) + @EqualsAndHashCode(callSuper = true) + @FieldNameConstants + public static class FormConfigFullRepresentation extends FormConfigOverviewRepresentation { + + /** + * The groups this config is shared with. + */ + private Collection groups; + + private JsonNode values; + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/export/AbsExportGenerator.java b/backend/src/main/java/com/bakdata/conquery/models/forms/export/AbsExportGenerator.java index e69de29bb2d..8b137891791 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/export/AbsExportGenerator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/export/AbsExportGenerator.java @@ -0,0 +1 @@ + diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/export/RelExportGenerator.java b/backend/src/main/java/com/bakdata/conquery/models/forms/export/RelExportGenerator.java index b3e78b8752c..b82b6c6eaab 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/export/RelExportGenerator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/export/RelExportGenerator.java @@ -14,28 +14,43 @@ @AllArgsConstructor public class RelExportGenerator { - - public static RelativeFormQuery generate(RelativeMode mode) { - List - resolutionsAndAlignments = - ExportForm.getResolutionAlignmentMap(mode.getForm().getResolvedResolutions(), mode.getTimeUnit().getAlignment()); + public static RelativeFormQuery generate(RelativeMode mode) { - return generate(mode.getForm() - .getPrerequisite(), mode.getResolvedFeatures(), mode.getIndexSelector(), mode.getIndexPlacement(), mode.getTimeCountBefore(), mode.getTimeCountAfter(), mode.getTimeUnit(), resolutionsAndAlignments); + List resolutionsAndAlignments = ExportForm.getResolutionAlignmentMap( + mode.getForm().getResolvedResolutions(), + mode.getTimeUnit().getAlignment()); + + return generate( + mode.getForm().getPrerequisite(), + mode.getResolvedFeatures(), + mode.getIndexSelector(), + mode.getIndexPlacement(), + mode.getTimeCountBefore(), + mode.getTimeCountAfter(), + mode.getTimeUnit(), + resolutionsAndAlignments); } - public static RelativeFormQuery generate(Query query, ArrayConceptQuery features, TemporalSamplerFactory indexSelector, IndexPlacement indexPlacement, int timeCountBefore, int timeCountAfter, CalendarUnit timeUnit, List resolutionsAndAlignments) { + public static RelativeFormQuery generate( + Query query, + ArrayConceptQuery features, + TemporalSamplerFactory indexSelector, + IndexPlacement indexPlacement, + int timeCountBefore, + int timeCountAfter, + CalendarUnit timeUnit, + List resolutionsAndAlignments) { return new RelativeFormQuery( - query, - features, - indexSelector, - indexPlacement, - timeCountBefore, - timeCountAfter, - timeUnit, - resolutionsAndAlignments + query, + features, + indexSelector, + indexPlacement, + timeCountBefore, + timeCountAfter, + timeUnit, + resolutionsAndAlignments ); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java b/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java index a128ba8e671..82c39dae4d9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java @@ -1,190 +1,197 @@ -package com.bakdata.conquery.models.forms.frontendconfiguration; - -import java.time.Instant; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Stream; -import jakarta.inject.Inject; -import jakarta.validation.Validator; - -import com.bakdata.conquery.apiv1.FormConfigPatch; -import com.bakdata.conquery.apiv1.forms.FormConfigAPI; -import com.bakdata.conquery.io.jackson.Jackson; -import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.models.auth.entities.Subject; -import com.bakdata.conquery.models.auth.entities.User; -import com.bakdata.conquery.models.auth.permissions.Ability; -import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; -import com.bakdata.conquery.models.auth.permissions.FormConfigPermission; -import com.bakdata.conquery.models.auth.permissions.WildcardPermission; -import com.bakdata.conquery.models.exceptions.ValidatorHelper; -import com.bakdata.conquery.models.forms.configs.FormConfig; -import com.bakdata.conquery.models.forms.configs.FormConfig.FormConfigFullRepresentation; -import com.bakdata.conquery.models.forms.configs.FormConfig.FormConfigOverviewRepresentation; -import com.bakdata.conquery.models.identifiable.ids.NamespacedId; -import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; -import com.bakdata.conquery.models.identifiable.ids.specific.FormConfigId; -import com.bakdata.conquery.models.worker.DatasetRegistry; -import com.bakdata.conquery.models.worker.Namespace; -import com.bakdata.conquery.resources.api.FormConfigResource; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; -import org.jetbrains.annotations.TestOnly; - -/** - * Holds the logic that serves the endpoints defined in {@link FormConfigResource}. - */ -@Slf4j -@NoArgsConstructor -@AllArgsConstructor -public class FormConfigProcessor { - - @Getter(onMethod = @__({@TestOnly})) - private static final ObjectMapper - MAPPER = - Jackson.MAPPER.copy().disable(SerializationFeature.WRITE_EMPTY_JSON_ARRAYS, SerializationFeature.WRITE_NULL_MAP_VALUES); - @Inject - private Validator validator; - @Inject - private MetaStorage storage; - @Inject - private DatasetRegistry datasetRegistry; - - /** - * Return an overview of all form config available to the subject. The selection can be reduced by setting a specific formType. - * The provided overview does not contain the configured values for the form, just the meta data. - * - * @param subject The subject vor which the overview is created. - * @param dataset - * @param requestedFormType Optional form type to filter the overview to that specific type. - **/ - public Stream getConfigsByFormType(@NonNull Subject subject, DatasetId dataset, @NonNull Set requestedFormType) { - - if (requestedFormType.isEmpty()) { - // If no specific form type is provided, show all types the subject is permitted to create. - // However, if a subject queries for specific form types, we will show all matching regardless whether - // the form config can be used by the subject again. - final Set allowedFormTypes = new HashSet<>(); - - for (FormType formType : FormScanner.FRONTEND_FORM_CONFIGS.values()) { - if (!subject.isPermitted(formType, Ability.CREATE)) { - continue; - } - - allowedFormTypes.add(formType.getName()); - } - requestedFormType = allowedFormTypes; - } - - final Set formTypesFinal = requestedFormType; - - final Stream stream = storage.getAllFormConfigs() - .filter(c -> dataset.equals(c.getDataset())) - .filter(c -> formTypesFinal.contains(c.getFormType())) - .filter(c -> subject.isPermitted(c, Ability.READ)); - - - return stream.map(c -> c.overview(subject)); - } - - /** - * Returns the full configuration of a configuration (meta data + configured values). - * It also tried to convert all {@link NamespacedId}s into the given dataset, so that the frontend can resolve them. - */ - public FormConfigFullRepresentation getConfig(Subject subject, FormConfigId formId) { - final FormConfig form = formId.resolve(); - - subject.authorize(form, Ability.READ); - return form.fullRepresentation(storage, subject); - } - - /** - * Adds the provided config to the desired dataset and the datasets that the - * subject has access to (has the READ ability on the Dataset), if the config is - * translatable to those. - */ - public FormConfig addConfig(Subject subject, DatasetId targetDataset, FormConfigAPI config) { - - final Namespace namespace = datasetRegistry.get(targetDataset); - - subject.authorize(namespace.getDataset(), Ability.READ); - - final FormConfig internalConfig = config.intern(subject.getId(), targetDataset); - // Add the config immediately to the submitted dataset - addConfigToDataset(internalConfig); - - return internalConfig; - } - - /** - * Adds a formular configuration under a specific dataset to the storage and grants the user the rights to manage/patch it. - */ - private void addConfigToDataset(FormConfig internalConfig) { - ValidatorHelper.failOnError(log, validator.validate(internalConfig)); - - internalConfig.setMetaStorage(storage); - storage.addFormConfig(internalConfig); - } - - /** - * Applies a patch to a configuration that allows to change its label or tags or even share it. - */ - public FormConfigFullRepresentation patchConfig(Subject subject, FormConfigId configId, FormConfigPatch patch) { - FormConfig config = configId.resolve(); - - patch.applyTo(config, storage, subject); - - storage.updateFormConfig(config); - - return config.fullRepresentation(storage, subject); - } - - /** - * Deletes a configuration from the storage and all permissions, that have this configuration as target. - */ - public void deleteConfig(Subject subject, FormConfigId configId) { - final User user = subject.getUser(); - - user.authorize(configId, Ability.DELETE); - storage.removeFormConfig(configId); - // Delete corresponding permissions (Maybe better to put it into a slow job) - for (ConqueryPermission permission : user.getPermissions()) { - - final WildcardPermission wpermission = (WildcardPermission) permission; - - if (!wpermission.getDomains().contains(FormConfigPermission.DOMAIN.toLowerCase())) { - continue; - } - if (!wpermission.getInstances().contains(configId.toString().toLowerCase())) { - continue; - } - - if (!wpermission.getInstances().isEmpty()) { - // Create new permission if it was a composite permission - final Set instancesCleared = new HashSet<>(wpermission.getInstances()); - instancesCleared.remove(configId.toString()); - final WildcardPermission clearedPermission = - new WildcardPermission(List.of(wpermission.getDomains(), wpermission.getAbilities(), instancesCleared), Instant.now()); - user.addPermission(clearedPermission); - } - - user.removePermission(wpermission); - } - } - - /** - * Simple container so that the frontend receives an JSON object instead of a string. - */ - @Data - @AllArgsConstructor - public static class PostResponse { - private FormConfigId id; - } -} +package com.bakdata.conquery.models.forms.frontendconfiguration; + +import jakarta.inject.Inject; +import jakarta.validation.Validator; +import java.time.Instant; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; + +import com.bakdata.conquery.apiv1.FormConfigPatch; +import com.bakdata.conquery.apiv1.forms.FormConfigAPI; +import com.bakdata.conquery.io.jackson.Jackson; +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.models.auth.entities.Subject; +import com.bakdata.conquery.models.auth.entities.User; +import com.bakdata.conquery.models.auth.permissions.Ability; +import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; +import com.bakdata.conquery.models.auth.permissions.FormConfigPermission; +import com.bakdata.conquery.models.auth.permissions.WildcardPermission; +import com.bakdata.conquery.models.exceptions.ValidatorHelper; +import com.bakdata.conquery.models.forms.configs.FormConfig; +import com.bakdata.conquery.models.forms.configs.FormConfig.FormConfigFullRepresentation; +import com.bakdata.conquery.models.forms.configs.FormConfig.FormConfigOverviewRepresentation; +import com.bakdata.conquery.models.identifiable.ids.NamespacedId; +import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; +import com.bakdata.conquery.models.identifiable.ids.specific.FormConfigId; +import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.Namespace; +import com.bakdata.conquery.resources.api.FormConfigResource; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.TestOnly; + +/** + * Holds the logic that serves the endpoints defined in {@link FormConfigResource}. + */ +@Slf4j +@NoArgsConstructor +@AllArgsConstructor +public class FormConfigProcessor { + + @Getter(onMethod = @__({@TestOnly})) + private static final ObjectMapper MAPPER = Jackson.MAPPER.copy() + .disable( + SerializationFeature.WRITE_EMPTY_JSON_ARRAYS, + SerializationFeature.WRITE_NULL_MAP_VALUES); + @Inject + private Validator validator; + @Inject + private MetaStorage storage; + @Inject + private DatasetRegistry datasetRegistry; + + /** + * Return an overview of all form config available to the subject. The selection can be reduced by setting a specific formType. + * The provided overview does not contain the configured values for the form, just the meta data. + * + * @param subject The subject vor which the overview is created. + * @param dataset + * @param requestedFormType Optional form type to filter the overview to that specific type. + **/ + public Stream getConfigsByFormType( + @NonNull Subject subject, + DatasetId dataset, + @NonNull Set requestedFormType) { + + if (requestedFormType.isEmpty()) { + // If no specific form type is provided, show all types the subject is permitted to create. + // However, if a subject queries for specific form types, we will show all matching regardless whether + // the form config can be used by the subject again. + final Set allowedFormTypes = new HashSet<>(); + + for (FormType formType : FormScanner.FRONTEND_FORM_CONFIGS.values()) { + if (!subject.isPermitted(formType, Ability.CREATE)) { + continue; + } + + allowedFormTypes.add(formType.getName()); + } + requestedFormType = allowedFormTypes; + } + + final Set formTypesFinal = requestedFormType; + + final Stream stream = storage.getAllFormConfigs() + .filter( + c -> dataset.equals(c.getDataset())) + .filter(c -> formTypesFinal.contains(c.getFormType())) + .filter( + c -> subject.isPermitted(c, Ability.READ)); + + + return stream.map(c -> c.overview(subject)); + } + + /** + * Returns the full configuration of a configuration (meta data + configured values). + * It also tried to convert all {@link NamespacedId}s into the given dataset, so that the frontend can resolve them. + */ + public FormConfigFullRepresentation getConfig(Subject subject, FormConfigId formId) { + final FormConfig form = formId.resolve(); + + subject.authorize(form, Ability.READ); + return form.fullRepresentation(storage, subject); + } + + /** + * Adds the provided config to the desired dataset and the datasets that the + * subject has access to (has the READ ability on the Dataset), if the config is + * translatable to those. + */ + public FormConfig addConfig(Subject subject, DatasetId targetDataset, FormConfigAPI config) { + + final Namespace namespace = datasetRegistry.get(targetDataset); + + subject.authorize(namespace.getDataset(), Ability.READ); + + final FormConfig internalConfig = config.intern(subject.getId(), targetDataset); + // Add the config immediately to the submitted dataset + addConfigToDataset(internalConfig); + + return internalConfig; + } + + /** + * Adds a formular configuration under a specific dataset to the storage and grants the user the rights to manage/patch it. + */ + private void addConfigToDataset(FormConfig internalConfig) { + ValidatorHelper.failOnError(log, validator.validate(internalConfig)); + + internalConfig.setMetaStorage(storage); + storage.addFormConfig(internalConfig); + } + + /** + * Applies a patch to a configuration that allows to change its label or tags or even share it. + */ + public FormConfigFullRepresentation patchConfig(Subject subject, FormConfigId configId, FormConfigPatch patch) { + FormConfig config = configId.resolve(); + + patch.applyTo(config, storage, subject); + + storage.updateFormConfig(config); + + return config.fullRepresentation(storage, subject); + } + + /** + * Deletes a configuration from the storage and all permissions, that have this configuration as target. + */ + public void deleteConfig(Subject subject, FormConfigId configId) { + final User user = subject.getUser(); + + user.authorize(configId, Ability.DELETE); + storage.removeFormConfig(configId); + // Delete corresponding permissions (Maybe better to put it into a slow job) + for (ConqueryPermission permission : user.getPermissions()) { + + final WildcardPermission wpermission = (WildcardPermission) permission; + + if (!wpermission.getDomains().contains(FormConfigPermission.DOMAIN.toLowerCase())) { + continue; + } + if (!wpermission.getInstances().contains(configId.toString().toLowerCase())) { + continue; + } + + if (!wpermission.getInstances().isEmpty()) { + // Create new permission if it was a composite permission + final Set instancesCleared = new HashSet<>(wpermission.getInstances()); + instancesCleared.remove(configId.toString()); + final WildcardPermission clearedPermission = new WildcardPermission( + List.of(wpermission.getDomains(), wpermission.getAbilities(), instancesCleared), + Instant.now()); + user.addPermission(clearedPermission); + } + + user.removePermission(wpermission); + } + } + + /** + * Simple container so that the frontend receives an JSON object instead of a string. + */ + @Data + @AllArgsConstructor + public static class PostResponse { + private FormConfigId id; + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormScanner.java b/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormScanner.java index 04e6bb86546..e9ba3f17b38 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormScanner.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormScanner.java @@ -98,7 +98,8 @@ private Map generateFEFormConfigMap() { final List frontendConfigs = findFrontendFormConfigs(); // Match frontend form configurations to backend implementations - final ImmutableMap.Builder result = ImmutableMap.builderWithExpectedSize(frontendConfigs.size()); + final ImmutableMap.Builder result = ImmutableMap.builderWithExpectedSize( + frontendConfigs.size()); for (FormFrontendConfigInformation configInfo : frontendConfigs) { @@ -123,23 +124,26 @@ private Map generateFEFormConfigMap() { // Make relative manual URLs relative to configured handbook base // Config url mappings override urls from frontend config jsons final URI manualURL = config.getPluginConfig(ManualConfig.class) - // first check override - .map(ManualConfig::getForms) - .map(m -> m.get(fullTypeIdentifier)) - // then query the frontend config json - .orElseGet(() -> { - final JsonNode manualUrl = configTree.get(MANUAL_URL_KEY); - if (manualUrl == null) { - // Nothing specified, skip - return null; - } - if (!manualUrl.isTextual()) { - throw new IllegalArgumentException( - String.format("FrontendFormConfig %s contained field 'manualUrl' but it was not a text. Was: '%s'.", fullTypeIdentifier, manualUrl.getNodeType())); - } - - return URI.create(manualUrl.textValue()); - }); + // first check override + .map(ManualConfig::getForms) + .map(m -> m.get(fullTypeIdentifier)) + // then query the frontend config json + .orElseGet(() -> { + final JsonNode manualUrl = configTree.get(MANUAL_URL_KEY); + if (manualUrl == null) { + // Nothing specified, skip + return null; + } + if (!manualUrl.isTextual()) { + throw new IllegalArgumentException( + String.format( + "FrontendFormConfig %s contained field 'manualUrl' but it was not a text. Was: '%s'.", + fullTypeIdentifier, + manualUrl.getNodeType())); + } + + return URI.create(manualUrl.textValue()); + }); final URL manualBaseUrl = config.getFrontend().getManualUrl(); @@ -149,7 +153,9 @@ private Map generateFEFormConfigMap() { final TextNode manualNode = relativizeManualUrl(fullTypeIdentifier, manualURL, manualBaseUrl); if (manualNode == null) { - log.warn("Manual url relativization did not succeed for {}. Skipping registration.", fullTypeIdentifier); + log.warn( + "Manual url relativization did not succeed for {}. Skipping registration.", + fullTypeIdentifier); continue; } @@ -158,7 +164,11 @@ private Map generateFEFormConfigMap() { result.put(fullTypeIdentifier, new FormType(fullTypeIdentifier, configTree)); - log.info("Form[{}] from `{}` of Type[{}]", fullTypeIdentifier, configInfo.getOrigin(), forms.get(typeIdentifier).getName()); + log.info( + "Form[{}] from `{}` of Type[{}]", + fullTypeIdentifier, + configInfo.getOrigin(), + forms.get(typeIdentifier).getName()); } return result.build(); @@ -201,9 +211,11 @@ private List findFrontendFormConfigs() { try { formConfigProvider.addFormConfigs(frontendConfigs); - } - catch (Exception e) { - log.error("Unable to collect frontend form configurations from {}.", formConfigProvider.getProviderName(), e); + } catch (Exception e) { + log.error( + "Unable to collect frontend form configurations from {}.", + formConfigProvider.getProviderName(), + e); } } @@ -215,11 +227,17 @@ private static boolean validTypeId(JsonNode node) { return node != null && node.isTextual() && !node.asText().isEmpty(); } - private TextNode relativizeManualUrl(@NonNull String formTypeIdentifier, @NonNull URI manualUri, @NonNull URL manualBaseUrl) { + private TextNode relativizeManualUrl( + @NonNull String formTypeIdentifier, + @NonNull URI manualUri, + @NonNull URL manualBaseUrl) { try { if (manualUri.isAbsolute()) { - log.trace("Manual url for {} was already absolute: {}. Skipping relativization.", formTypeIdentifier, manualUri); + log.trace( + "Manual url for {} was already absolute: {}. Skipping relativization.", + formTypeIdentifier, + manualUri); return new TextNode(manualUri.toURL().toString()); } @@ -227,13 +245,15 @@ private TextNode relativizeManualUrl(@NonNull String formTypeIdentifier, @NonNul final String absoluteUrl = manualBaseUrl.toURI().resolve(manualUri).toURL().toString(); log.trace("Computed manual url for {}: {}", formTypeIdentifier, absoluteUrl); return new TextNode(absoluteUrl); - } - catch (URISyntaxException e) { - log.warn("Unable to resolve manual base url ('{}') and relative manual url ('{}')", manualBaseUrl, manualUri, e); + } catch (URISyntaxException e) { + log.warn( + "Unable to resolve manual base url ('{}') and relative manual url ('{}')", + manualBaseUrl, + manualUri, + e); return null; } - } - catch (MalformedURLException e) { + } catch (MalformedURLException e) { log.error("Unable to build url", e); return null; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/ResourceFormConfigProvider.java b/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/ResourceFormConfigProvider.java index 77c24a6d692..4b22f57eddc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/ResourceFormConfigProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/ResourceFormConfigProvider.java @@ -21,24 +21,29 @@ public class ResourceFormConfigProvider { public static void accept(ImmutableCollection.Builder formConfigInfos) { - ResourceList frontendConfigs = CPSTypeIdResolver.SCAN_RESULT - .getResourcesMatchingPattern(Pattern.compile(".*\\.frontend_conf\\.json")); + ResourceList frontendConfigs = CPSTypeIdResolver.SCAN_RESULT.getResourcesMatchingPattern( + Pattern.compile(".*\\.frontend_conf\\.json")); for (Resource config : frontendConfigs) { try (config) { try (InputStream in = config.open()) { JsonNode configTree = Jackson.MAPPER.reader().readTree(in); if (!configTree.isObject()) { - log.warn("Expected '{}' to be an JSON object but was '{}'. Skipping registration.", config.getPath(), configTree.getNodeType()); + log.warn( + "Expected '{}' to be an JSON object but was '{}'. Skipping registration.", + config.getPath(), + configTree.getNodeType()); } - formConfigInfos.add(new FormFrontendConfigInformation("Resource " + config.getPath(), (ObjectNode) configTree)); + formConfigInfos.add( + new FormFrontendConfigInformation("Resource " + config.getPath(), (ObjectNode) configTree)); } - } - catch (IOException e) { - throw new IllegalArgumentException(String.format("Could not parse the frontend config: %s", config.getPath()), e); + } catch (IOException e) { + throw new IllegalArgumentException( + String.format("Could not parse the frontend config: %s", config.getPath()), + e); } } - + } - -} \ No newline at end of file + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQuery.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQuery.java index e734372a5d9..fb711813b04 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQuery.java @@ -1,115 +1,119 @@ -package com.bakdata.conquery.models.forms.managed; - -import java.time.LocalDate; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import java.util.function.Consumer; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotNull; - -import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; -import com.bakdata.conquery.apiv1.query.ArrayConceptQuery; -import com.bakdata.conquery.apiv1.query.Query; -import com.bakdata.conquery.apiv1.query.QueryDescription; -import com.bakdata.conquery.apiv1.query.ResultHeaders; -import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.models.common.Range; -import com.bakdata.conquery.models.common.daterange.CDateRange; -import com.bakdata.conquery.models.forms.util.DateContext; -import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; -import com.bakdata.conquery.models.query.DateAggregationMode; -import com.bakdata.conquery.models.query.QueryExecutionContext; -import com.bakdata.conquery.models.query.QueryPlanContext; -import com.bakdata.conquery.models.query.QueryResolveContext; -import com.bakdata.conquery.models.query.RequiredEntities; -import com.bakdata.conquery.models.query.Visitable; -import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import lombok.Getter; - -@Getter -@CPSType(id = "ABSOLUTE_FORM_QUERY", base = QueryDescription.class) -public class AbsoluteFormQuery extends Query { - - /** - * Index of the column, where the Resolutions name will be placed. - */ - public static final int RESOLUTION_INDEX = 0; - - /** - * Indef of the column, where the time periods will be placed. - */ - public static final int TIME_INDEX = 2; - - /** - * see {@linkplain #getResultInfos()}. - */ - public static final int FEATURES_OFFSET = 3; - - @NotNull - @Valid - private final Query query; - @NotNull - @Valid - private final Range dateRange; - @NotNull - @Valid - private final ArrayConceptQuery features; - @NotNull - private final List resolutionsAndAlignmentMap; - - public AbsoluteFormQuery(Query query, Range dateRange, ArrayConceptQuery features, List resolutionsAndAlignmentMap) { - this.query = query; - this.dateRange = dateRange; - this.features = features; - this.resolutionsAndAlignmentMap = resolutionsAndAlignmentMap; - } - - @Override - public void resolve(QueryResolveContext context) { - query.resolve(context); - features.resolve(context.withDateAggregationMode(DateAggregationMode.NONE)); - } - - @Override - public AbsoluteFormQueryPlan createQueryPlan(QueryPlanContext context) { - return new AbsoluteFormQueryPlan( - query.createQueryPlan(context), - new FormQueryPlan( - DateContext.generateAbsoluteContexts(CDateRange.of(dateRange), resolutionsAndAlignmentMap), - features.createQueryPlan(context), - false - ) - ); - } - - @Override - public List getResultInfos() { - final List resultInfos = new ArrayList<>(); - - resultInfos.add(ResultHeaders.formResolutionInfo()); - resultInfos.add(ResultHeaders.formContextInfo()); - resultInfos.add(ResultHeaders.formDateRangeInfo()); - resultInfos.addAll(features.getResultInfos()); - - return resultInfos; - } - - @Override - public void visit(Consumer visitor) { - visitor.accept(this); - query.visit(visitor); - features.visit(visitor); - } - - @Override - public void collectRequiredQueries(Set requiredQueries) { - query.collectRequiredQueries(requiredQueries); - features.collectRequiredQueries(requiredQueries); - } - - @Override - public RequiredEntities collectRequiredEntities(QueryExecutionContext context) { - return query.collectRequiredEntities(context); - } -} +package com.bakdata.conquery.models.forms.managed; + +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.function.Consumer; + +import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; +import com.bakdata.conquery.apiv1.query.ArrayConceptQuery; +import com.bakdata.conquery.apiv1.query.Query; +import com.bakdata.conquery.apiv1.query.QueryDescription; +import com.bakdata.conquery.apiv1.query.ResultHeaders; +import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.models.common.Range; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.forms.util.DateContext; +import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; +import com.bakdata.conquery.models.query.DateAggregationMode; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.QueryPlanContext; +import com.bakdata.conquery.models.query.QueryResolveContext; +import com.bakdata.conquery.models.query.RequiredEntities; +import com.bakdata.conquery.models.query.Visitable; +import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import lombok.Getter; + +@Getter +@CPSType(id = "ABSOLUTE_FORM_QUERY", base = QueryDescription.class) +public class AbsoluteFormQuery extends Query { + + /** + * Index of the column, where the Resolutions name will be placed. + */ + public static final int RESOLUTION_INDEX = 0; + + /** + * Indef of the column, where the time periods will be placed. + */ + public static final int TIME_INDEX = 2; + + /** + * see {@linkplain #getResultInfos()}. + */ + public static final int FEATURES_OFFSET = 3; + + @NotNull + @Valid + private final Query query; + @NotNull + @Valid + private final Range dateRange; + @NotNull + @Valid + private final ArrayConceptQuery features; + @NotNull + private final List resolutionsAndAlignmentMap; + + public AbsoluteFormQuery( + Query query, + Range dateRange, + ArrayConceptQuery features, + List resolutionsAndAlignmentMap) { + this.query = query; + this.dateRange = dateRange; + this.features = features; + this.resolutionsAndAlignmentMap = resolutionsAndAlignmentMap; + } + + @Override + public void resolve(QueryResolveContext context) { + query.resolve(context); + features.resolve(context.withDateAggregationMode(DateAggregationMode.NONE)); + } + + @Override + public AbsoluteFormQueryPlan createQueryPlan(QueryPlanContext context) { + return new AbsoluteFormQueryPlan( + query.createQueryPlan(context), + new FormQueryPlan( + DateContext.generateAbsoluteContexts(CDateRange.of(dateRange), resolutionsAndAlignmentMap), + features.createQueryPlan(context), + false + ) + ); + } + + @Override + public List getResultInfos() { + final List resultInfos = new ArrayList<>(); + + resultInfos.add(ResultHeaders.formResolutionInfo()); + resultInfos.add(ResultHeaders.formContextInfo()); + resultInfos.add(ResultHeaders.formDateRangeInfo()); + resultInfos.addAll(features.getResultInfos()); + + return resultInfos; + } + + @Override + public void visit(Consumer visitor) { + visitor.accept(this); + query.visit(visitor); + features.visit(visitor); + } + + @Override + public void collectRequiredQueries(Set requiredQueries) { + query.collectRequiredQueries(requiredQueries); + features.collectRequiredQueries(requiredQueries); + } + + @Override + public RequiredEntities collectRequiredEntities(QueryExecutionContext context) { + return query.collectRequiredEntities(context); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQueryPlan.java index 4224061a7b3..ff3c7364172 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQueryPlan.java @@ -12,8 +12,10 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; +import org.jspecify.annotations.NonNull; -@Getter @RequiredArgsConstructor +@Getter +@RequiredArgsConstructor @ToString public class AbsoluteFormQueryPlan implements QueryPlan { @@ -45,7 +47,7 @@ public boolean isOfInterest(Entity entity) { } @Override - public Optional> getValidityDateAggregator() { + public @NonNull Optional> getValidityDateAggregator() { return subPlan.getValidityDateAggregator(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQuery.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQuery.java index 126d52f77b0..2b4c6c33d44 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQuery.java @@ -1,114 +1,111 @@ -package com.bakdata.conquery.models.forms.managed; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Set; -import java.util.function.Consumer; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; - -import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; -import com.bakdata.conquery.apiv1.query.ArrayConceptQuery; -import com.bakdata.conquery.apiv1.query.Query; -import com.bakdata.conquery.apiv1.query.QueryDescription; -import com.bakdata.conquery.apiv1.query.ResultHeaders; -import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; -import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.models.common.daterange.CDateRange; -import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; -import com.bakdata.conquery.models.query.DateAggregationMode; -import com.bakdata.conquery.models.query.QueryExecutionContext; -import com.bakdata.conquery.models.query.QueryPlanContext; -import com.bakdata.conquery.models.query.QueryResolveContext; -import com.bakdata.conquery.models.query.RequiredEntities; -import com.bakdata.conquery.models.query.Visitable; -import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.fasterxml.jackson.annotation.JsonCreator; -import lombok.Getter; -import lombok.RequiredArgsConstructor; - -/** - * This query uses the date range defined by {@link EntityDateQuery#query} for each entity and applies it as a - * date restriction for the following query defined by {@link EntityDateQuery#features}. - * The additional {@link EntityDateQuery#dateRange} is applied globally on all entities. - */ -@CPSType(id = "ENTITY_DATE_QUERY", base = QueryDescription.class) -@Getter -@RequiredArgsConstructor(onConstructor_ = {@JsonCreator}) -public class EntityDateQuery extends Query { - - @NotNull - @Valid - private final Query query; - @NotNull - @Valid - private final ArrayConceptQuery features; - - @NotNull - @NotEmpty - private final List resolutionsAndAlignments; - - private final CDateRange dateRange; - - @NotNull - private final DateAggregationMode dateAggregationMode; - - - @Override - public EntityDateQueryPlan createQueryPlan(QueryPlanContext context) { - // Clear all selects we need only the date union which is enforced through the content - Visitable.stream(query) - .filter(CQConcept.class::isInstance) - .map(CQConcept.class::cast) - .forEach(concept -> { - concept.setSelects(Collections.emptyList()); - concept.getTables().forEach(t -> t.setSelects(Collections.emptyList())); - }); - - return new EntityDateQueryPlan( - query.createQueryPlan(context), - features.createQueryPlan(context), - resolutionsAndAlignments, - dateRange - ); - } - - @Override - public void collectRequiredQueries(Set requiredQueries) { - query.collectRequiredQueries(requiredQueries); - features.collectRequiredQueries(requiredQueries); - } - - @Override - public void resolve(QueryResolveContext context) { - query.resolve(context.withDateAggregationMode(dateAggregationMode)); - features.resolve(context); - } - - @Override - public List getResultInfos() { - List resultInfos = new ArrayList<>(); - resultInfos.add(ResultHeaders.formResolutionInfo()); - resultInfos.add(ResultHeaders.formContextInfo()); - resultInfos.add(ResultHeaders.formDateRangeInfo()); - - resultInfos.addAll(features.getResultInfos()); - - return resultInfos; - - } - - @Override - public void visit(Consumer visitor) { - visitor.accept(this); - query.visit(visitor); - features.visit(visitor); - } - - @Override - public RequiredEntities collectRequiredEntities(QueryExecutionContext context) { - return query.collectRequiredEntities(context); - } -} +package com.bakdata.conquery.models.forms.managed; + +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.function.Consumer; + +import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; +import com.bakdata.conquery.apiv1.query.ArrayConceptQuery; +import com.bakdata.conquery.apiv1.query.Query; +import com.bakdata.conquery.apiv1.query.QueryDescription; +import com.bakdata.conquery.apiv1.query.ResultHeaders; +import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; +import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; +import com.bakdata.conquery.models.query.DateAggregationMode; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.QueryPlanContext; +import com.bakdata.conquery.models.query.QueryResolveContext; +import com.bakdata.conquery.models.query.RequiredEntities; +import com.bakdata.conquery.models.query.Visitable; +import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.fasterxml.jackson.annotation.JsonCreator; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +/** + * This query uses the date range defined by {@link EntityDateQuery#query} for each entity and applies it as a + * date restriction for the following query defined by {@link EntityDateQuery#features}. + * The additional {@link EntityDateQuery#dateRange} is applied globally on all entities. + */ +@CPSType(id = "ENTITY_DATE_QUERY", base = QueryDescription.class) +@Getter +@RequiredArgsConstructor(onConstructor_ = {@JsonCreator}) +public class EntityDateQuery extends Query { + + @NotNull + @Valid + private final Query query; + @NotNull + @Valid + private final ArrayConceptQuery features; + + @NotNull + @NotEmpty + private final List resolutionsAndAlignments; + + private final CDateRange dateRange; + + @NotNull + private final DateAggregationMode dateAggregationMode; + + + @Override + public EntityDateQueryPlan createQueryPlan(QueryPlanContext context) { + // Clear all selects we need only the date union which is enforced through the content + Visitable.stream(query).filter(CQConcept.class::isInstance).map(CQConcept.class::cast).forEach(concept -> { + concept.setSelects(Collections.emptyList()); + concept.getTables().forEach(t -> t.setSelects(Collections.emptyList())); + }); + + return new EntityDateQueryPlan( + query.createQueryPlan(context), + features.createQueryPlan(context), + resolutionsAndAlignments, + dateRange + ); + } + + @Override + public void collectRequiredQueries(Set requiredQueries) { + query.collectRequiredQueries(requiredQueries); + features.collectRequiredQueries(requiredQueries); + } + + @Override + public void resolve(QueryResolveContext context) { + query.resolve(context.withDateAggregationMode(dateAggregationMode)); + features.resolve(context); + } + + @Override + public List getResultInfos() { + List resultInfos = new ArrayList<>(); + resultInfos.add(ResultHeaders.formResolutionInfo()); + resultInfos.add(ResultHeaders.formContextInfo()); + resultInfos.add(ResultHeaders.formDateRangeInfo()); + + resultInfos.addAll(features.getResultInfos()); + + return resultInfos; + + } + + @Override + public void visit(Consumer visitor) { + visitor.accept(this); + query.visit(visitor); + features.visit(visitor); + } + + @Override + public RequiredEntities collectRequiredEntities(QueryExecutionContext context) { + return query.collectRequiredEntities(context); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQueryPlan.java index 77218e9cdfe..7f1b234595a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQueryPlan.java @@ -1,81 +1,82 @@ -package com.bakdata.conquery.models.forms.managed; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - -import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; -import com.bakdata.conquery.models.common.CDateSet; -import com.bakdata.conquery.models.common.daterange.CDateRange; -import com.bakdata.conquery.models.forms.util.DateContext; -import com.bakdata.conquery.models.query.QueryExecutionContext; -import com.bakdata.conquery.models.query.entity.Entity; -import com.bakdata.conquery.models.query.queryplan.ArrayConceptQueryPlan; -import com.bakdata.conquery.models.query.queryplan.DateAggregator; -import com.bakdata.conquery.models.query.queryplan.QueryPlan; -import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.results.EntityResult; -import com.bakdata.conquery.models.query.results.MultilineEntityResult; -import lombok.RequiredArgsConstructor; -import lombok.ToString; - -/** - * Implementation of the QueryPlan for an {@link EntityDateQuery}. - */ -@RequiredArgsConstructor -@ToString -public class EntityDateQueryPlan implements QueryPlan { - - - private final QueryPlan query; - private final ArrayConceptQueryPlan features; - private final List resolutionsAndAlignments; - private final CDateRange dateRestriction; - - - @Override - public void init(QueryExecutionContext ctxt, Entity entity) { - query.init(ctxt, entity); - features.init(ctxt, entity); - } - - @Override - public Optional execute(QueryExecutionContext ctx, Entity entity) { - - // Don't set the query date aggregator here because the subqueries should set their aggregator independently - - // Execute the prerequisite query - Optional preResult = query.execute(ctx, entity); - if (preResult.isEmpty()) { - return Optional.empty(); - } - - Optional validityDateAggregator = query.getValidityDateAggregator(); - if (validityDateAggregator.isEmpty()) { - return Optional.empty(); - } - - final CDateSet aggregationResult = validityDateAggregator.get().createAggregationResult(); - aggregationResult.retainAll(dateRestriction); - - // Generate DateContexts in the provided resolutions - List contexts = new ArrayList<>(); - for (CDateRange range : aggregationResult.asRanges()) { - contexts.addAll(DateContext.generateAbsoluteContexts(range, resolutionsAndAlignments)); - } - - FormQueryPlan resolutionQuery = new FormQueryPlan(contexts, features, false); - - return resolutionQuery.execute(ctx, entity); - } - - @Override - public boolean isOfInterest(Entity entity) { - return query.isOfInterest(entity); - } - - @Override - public Optional> getValidityDateAggregator() { - return query.getValidityDateAggregator(); - } -} +package com.bakdata.conquery.models.forms.managed; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; +import com.bakdata.conquery.models.common.CDateSet; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.forms.util.DateContext; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; +import com.bakdata.conquery.models.query.queryplan.ArrayConceptQueryPlan; +import com.bakdata.conquery.models.query.queryplan.DateAggregator; +import com.bakdata.conquery.models.query.queryplan.QueryPlan; +import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; +import com.bakdata.conquery.models.query.results.EntityResult; +import com.bakdata.conquery.models.query.results.MultilineEntityResult; +import lombok.RequiredArgsConstructor; +import lombok.ToString; +import org.jspecify.annotations.NonNull; + +/** + * Implementation of the QueryPlan for an {@link EntityDateQuery}. + */ +@RequiredArgsConstructor +@ToString +public class EntityDateQueryPlan implements QueryPlan { + + + private final QueryPlan query; + private final ArrayConceptQueryPlan features; + private final List resolutionsAndAlignments; + private final CDateRange dateRestriction; + + + @Override + public void init(QueryExecutionContext ctxt, Entity entity) { + query.init(ctxt, entity); + features.init(ctxt, entity); + } + + @Override + public Optional execute(QueryExecutionContext ctx, Entity entity) { + + // Don't set the query date aggregator here because the subqueries should set their aggregator independently + + // Execute the prerequisite query + Optional preResult = query.execute(ctx, entity); + if (preResult.isEmpty()) { + return Optional.empty(); + } + + Optional validityDateAggregator = query.getValidityDateAggregator(); + if (validityDateAggregator.isEmpty()) { + return Optional.empty(); + } + + final CDateSet aggregationResult = validityDateAggregator.get().createAggregationResult(); + aggregationResult.retainAll(dateRestriction); + + // Generate DateContexts in the provided resolutions + List contexts = new ArrayList<>(); + for (CDateRange range : aggregationResult.asRanges()) { + contexts.addAll(DateContext.generateAbsoluteContexts(range, resolutionsAndAlignments)); + } + + FormQueryPlan resolutionQuery = new FormQueryPlan(contexts, features, false); + + return resolutionQuery.execute(ctx, entity); + } + + @Override + public boolean isOfInterest(Entity entity) { + return query.isOfInterest(entity); + } + + @Override + public @NonNull Optional> getValidityDateAggregator() { + return query.getValidityDateAggregator(); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ExternalExecution.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ExternalExecution.java index 8d50439658e..d3620fb62b9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ExternalExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ExternalExecution.java @@ -58,7 +58,13 @@ public class ExternalExecution extends ManagedForm { private UUID externalTaskId; - public ExternalExecution(ExternalForm form, UserId user, DatasetId dataset, MetaStorage metaStorage, DatasetRegistry datasetRegistry, ConqueryConfig config) { + public ExternalExecution( + ExternalForm form, + UserId user, + DatasetId dataset, + MetaStorage metaStorage, + DatasetRegistry datasetRegistry, + ConqueryConfig config) { super(form, user, dataset, metaStorage, datasetRegistry, config); } @@ -79,8 +85,10 @@ public void start() { // Check after possible sync final boolean isRunning = getExecutionManager().tryGetExecutionInfo(getId()) - .map(ExecutionManager.ExecutionInfo::getExecutionState) - .map(ExecutionState.RUNNING::equals).orElse(false); + .map( + ExecutionManager.ExecutionInfo::getExecutionState) + .map(ExecutionState.RUNNING::equals) + .orElse(false); if (isRunning) { throw new ConqueryError.ExecutionProcessingError(); } @@ -90,16 +98,23 @@ public void start() { final Dataset dataset = getNamespace().getDataset(); final User originalUser = getOwner().resolve(); final FormBackendConfig formBackendConfig = getConfig().getPluginConfigs(FormBackendConfig.class) - .filter(c -> c.supportsFormType(getSubmittedForm().getFormType())) - .collect(MoreCollectors.onlyElement()); + .filter( + c -> c.supportsFormType(getSubmittedForm().getFormType())) + .collect(MoreCollectors.onlyElement()); final User serviceUser = formBackendConfig.createServiceUser(originalUser, dataset); final ExternalFormBackendApi api = formBackendConfig.createApi(); super.start(); - getExecutionManager().addState(getId(), new ExternalExecutionInfoImpl(ExecutionState.RUNNING, new CountDownLatch(0), api, serviceUser)); + getExecutionManager().addState( + getId(), + new ExternalExecutionInfoImpl(ExecutionState.RUNNING, new CountDownLatch(0), api, serviceUser)); - final ExternalTaskState externalTaskState = api.postForm(getSubmitted(), originalUser, serviceUser, dataset); + final ExternalTaskState externalTaskState = api.postForm( + getSubmitted(), + originalUser, + serviceUser, + dataset); externalTaskId = externalTaskState.getId(); } } @@ -120,7 +135,8 @@ private void updateStatus(ExternalTaskState formState) { case RUNNING -> setProgress(formState.getProgress().floatValue()); case FAILURE -> fail(formState.getError()); case SUCCESS -> { - final List> resultsAssetMap = registerResultAssets(formState); + final List> resultsAssetMap = registerResultAssets( + formState); final ExternalExecutionInfo state = getExecutionManager().getExecutionInfo(getId()); state.setResultsAssetMap(resultsAssetMap); finish(ExecutionState.DONE); @@ -129,7 +145,8 @@ private void updateStatus(ExternalTaskState formState) { } } - private List> registerResultAssets(ExternalTaskState response) { + private List> registerResultAssets( + ExternalTaskState response) { final List> assetMap = new ArrayList<>(); response.getResults().forEach(asset -> assetMap.add(Pair.of(asset, createResultAssetBuilder(asset)))); return assetMap; @@ -156,10 +173,12 @@ public synchronized void finish(ExecutionState executionState) { private ExternalExecutionInfo.AssetBuilder createResultAssetBuilder(ResultAsset asset) { return (uriBuilder) -> { try { - final URI externalDownloadURL = ResultExternalResource.getDownloadURL(uriBuilder.clone(), this, asset.getAssetId()); + final URI externalDownloadURL = ResultExternalResource.getDownloadURL( + uriBuilder.clone(), + this, + asset.getAssetId()); return new ResultAsset(asset.label(), externalDownloadURL); - } - catch (URISyntaxException e) { + } catch (URISyntaxException e) { throw new RuntimeException(e); } }; diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/FormQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/FormQueryPlan.java index d07db86cbf2..659cde1a4a9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/FormQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/FormQueryPlan.java @@ -20,6 +20,7 @@ import lombok.Getter; import lombok.ToString; import lombok.extern.slf4j.Slf4j; +import org.jspecify.annotations.NonNull; @Getter @ToString @@ -33,7 +34,7 @@ public class FormQueryPlan implements QueryPlan { private final boolean withRelativeEventDate; private final boolean withObservationScope; - public FormQueryPlan(List dateContexts, ArrayConceptQueryPlan features, boolean withObservationScope ) { + public FormQueryPlan(List dateContexts, ArrayConceptQueryPlan features, boolean withObservationScope) { this.dateContexts = dateContexts; this.features = features; this.withObservationScope = withObservationScope; @@ -53,7 +54,8 @@ public FormQueryPlan(List dateContexts, ArrayConceptQueryPlan featu for (DateContext dateContext : dateContexts) { if ((dateContext.getEventDate() == null) == withRelativeEventDate) { - throw new IllegalStateException("QueryPlan has absolute AND relative date contexts. Only one kind is allowed."); + throw new IllegalStateException( + "QueryPlan has absolute AND relative date contexts. Only one kind is allowed."); } } constantCount = calculateConstantCount(withRelativeEventDate, this.withObservationScope); @@ -86,8 +88,9 @@ public Optional execute(QueryExecutionContext ctx, Entity dateRestriction.retainAll(dateContext.getDateRange()); // Reference the dates per sub-query, don't accumulate dates of all sub-queries - QueryExecutionContext innerContext = QueryUtils.determineDateAggregatorForContext(ctx, features::getValidityDateAggregator) - .withDateRestriction(dateRestriction); + QueryExecutionContext innerContext = QueryUtils.determineDateAggregatorForContext( + ctx, + features::getValidityDateAggregator).withDateRestriction(dateRestriction); features.init(ctx, entity); @@ -99,12 +102,12 @@ public Optional execute(QueryExecutionContext ctx, Entity } resultValues.addAll( - ResultModifier.modify( - subResult.get(), - ResultModifier.existAggValuesSetterFor(features.getAggregators(), OptionalInt.of(0)) - .unaryAndThen(v -> addConstants(v, dateContext)) - ) - .listResultLines() + ResultModifier.modify( + subResult.get(), + ResultModifier.existAggValuesSetterFor(features.getAggregators(), OptionalInt.of(0)) + .unaryAndThen( + v -> addConstants(v, dateContext)) + ).listResultLines() ); } @@ -122,9 +125,10 @@ private MultilineEntityResult createResultForNotContained(Entity entity, DateCon result.add(new Object[features.getAggregatorSize()]); return ResultModifier.modify( - new MultilineEntityResult(entity.getId(), result), - ResultModifier.existAggValuesSetterFor(getAggregators(), OptionalInt.of(0)) - .unaryAndThen(v -> addConstants(v, dateContext)) + new MultilineEntityResult(entity.getId(), result), + ResultModifier.existAggValuesSetterFor(getAggregators(), OptionalInt.of(0)) + .unaryAndThen( + v -> addConstants(v, dateContext)) ); } @@ -154,7 +158,7 @@ private Object[] addConstants(Object[] values, DateContext dateContext) { //add date range at [2] or [3] result[getDateRangeResultPosition()] = dateContext.getDateRange(); - if(withObservationScope){ + if (withObservationScope) { //add observation scope at [3] or [4] result[getDateRangeResultPosition() + 1] = dateContext.getFeatureGroup(); } @@ -163,7 +167,7 @@ private Object[] addConstants(Object[] values, DateContext dateContext) { } private int getDateRangeResultPosition() { - return constantCount - (withObservationScope ? 2 : 1 ); + return constantCount - (withObservationScope ? 2 : 1); } @Override @@ -173,7 +177,7 @@ public boolean isOfInterest(Entity entity) { } @Override - public Optional> getValidityDateAggregator() { + public @NonNull Optional> getValidityDateAggregator() { return Optional.empty(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedForm.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedForm.java index 6de2b2595f0..58e4eee3bb5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedForm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedForm.java @@ -1,95 +1,103 @@ -package com.bakdata.conquery.models.forms.managed; - -import java.util.function.Consumer; - -import com.bakdata.conquery.apiv1.forms.Form; -import com.bakdata.conquery.apiv1.forms.FormConfigAPI; -import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.execution.ManagedExecution; -import com.bakdata.conquery.models.forms.configs.FormConfig; -import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; -import com.bakdata.conquery.models.identifiable.ids.specific.UserId; -import com.bakdata.conquery.models.query.PrintSettings; -import com.bakdata.conquery.models.query.Visitable; -import com.bakdata.conquery.models.worker.DatasetRegistry; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.databind.DatabindContext; -import lombok.AccessLevel; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.ToString; -import lombok.extern.slf4j.Slf4j; - -/** - * Internal runtime representation of a form query. - */ -@ToString -@Slf4j -@EqualsAndHashCode(callSuper = true) -@CPSType(id = "MANAGED_FORM", base = ManagedExecution.class) -@NoArgsConstructor(access = AccessLevel.PROTECTED) -public abstract class ManagedForm extends ManagedExecution { - - /** - * The submitted form for this execution. - * - * @implNote We use the type {@link Form} here rather than the type parameter F. - * Using F causes the class to have a concrete type at runtime which in turn skips - * the object inspection of Jackson to look at the actual type member of the object (see {@link com.bakdata.conquery.io.cps.CPSTypeIdResolver#typeFromId(DatabindContext, String)}). - * This causes a problem, when the object uses types with {@link com.bakdata.conquery.io.cps.SubTyped}, - * as the subtype is only added to the {@link com.fasterxml.jackson.databind.DeserializationContext}, when the - * type is derived from the 
type
member not when Jackson can just infer the deserializer from the type of - * this property. - */ - @Getter - private Form submittedForm; - - protected ManagedForm(F submittedForm, UserId owner, DatasetId submittedDataset, MetaStorage storage, DatasetRegistry datasetRegistry, ConqueryConfig config) { - super(owner, submittedDataset, storage, datasetRegistry, config); - this.submittedForm = submittedForm; - } - - - @Override - public void start() { - synchronized (this) { - super.start(); - - if (getSubmittedForm().getValues() != null) { - // save as formConfig - final FormConfigAPI build = FormConfigAPI.builder().formType(getSubmittedForm().getFormType()) - .label(this.getLabelWithoutAutoLabelSuffix()) - .tags(this.getTags()) - .values(getSubmittedForm().getValues()).build(); - - final FormConfig formConfig = build.intern(getOwner(), getDataset()); - - getMetaStorage().addFormConfig(formConfig); - } - } - } - - @Override - public void visit(Consumer visitor) { - submittedForm.visit(visitor); - } - - - @Override - @JsonIgnore - public F getSubmitted() { - return (F) submittedForm; - } - - - - - @Override - protected String makeDefaultLabel(PrintSettings cfg) { - return getSubmittedForm().getLocalizedTypeLabel(); - } - -} +package com.bakdata.conquery.models.forms.managed; + +import java.util.function.Consumer; + +import com.bakdata.conquery.apiv1.forms.Form; +import com.bakdata.conquery.apiv1.forms.FormConfigAPI; +import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.execution.ManagedExecution; +import com.bakdata.conquery.models.forms.configs.FormConfig; +import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; +import com.bakdata.conquery.models.identifiable.ids.specific.UserId; +import com.bakdata.conquery.models.query.PrintSettings; +import com.bakdata.conquery.models.query.Visitable; +import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.DatabindContext; +import lombok.AccessLevel; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; + +/** + * Internal runtime representation of a form query. + */ +@ToString +@Slf4j +@EqualsAndHashCode(callSuper = true) +@CPSType(id = "MANAGED_FORM", base = ManagedExecution.class) +@NoArgsConstructor(access = AccessLevel.PROTECTED) +public abstract class ManagedForm extends ManagedExecution { + + /** + * The submitted form for this execution. + * + * @implNote We use the type {@link Form} here rather than the type parameter F. + * Using F causes the class to have a concrete type at runtime which in turn skips + * the object inspection of Jackson to look at the actual type member of the object (see {@link com.bakdata.conquery.io.cps.CPSTypeIdResolver#typeFromId(DatabindContext, String)}). + * This causes a problem, when the object uses types with {@link com.bakdata.conquery.io.cps.SubTyped}, + * as the subtype is only added to the {@link com.fasterxml.jackson.databind.DeserializationContext}, when the + * type is derived from the 
type
member not when Jackson can just infer the deserializer from the type of + * this property. + */ + @Getter + private Form submittedForm; + + protected ManagedForm( + F submittedForm, + UserId owner, + DatasetId submittedDataset, + MetaStorage storage, + DatasetRegistry datasetRegistry, + ConqueryConfig config) { + super(owner, submittedDataset, storage, datasetRegistry, config); + this.submittedForm = submittedForm; + } + + + @Override + public void start() { + synchronized (this) { + super.start(); + + if (getSubmittedForm().getValues() != null) { + // save as formConfig + final FormConfigAPI build = FormConfigAPI.builder() + .formType(getSubmittedForm().getFormType()) + .label( + this.getLabelWithoutAutoLabelSuffix()) + .tags(this.getTags()) + .values( + getSubmittedForm().getValues()) + .build(); + + final FormConfig formConfig = build.intern(getOwner(), getDataset()); + + getMetaStorage().addFormConfig(formConfig); + } + } + } + + @Override + public void visit(Consumer visitor) { + submittedForm.visit(visitor); + } + + + @Override + @JsonIgnore + public F getSubmitted() { + return (F) submittedForm; + } + + + @Override + protected String makeDefaultLabel(PrintSettings cfg) { + return getSubmittedForm().getLocalizedTypeLabel(); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java index 0aff7ed48f5..a50e2a87e59 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java @@ -64,7 +64,13 @@ public class ManagedInternalForm extends ManagedF @Getter(AccessLevel.PROTECTED) private Map initializedSubQueryHardRef; - public ManagedInternalForm(F form, UserId user, DatasetId submittedDataset, MetaStorage storage, DatasetRegistry datasetRegistry, ConqueryConfig config) { + public ManagedInternalForm( + F form, + UserId user, + DatasetId submittedDataset, + MetaStorage storage, + DatasetRegistry datasetRegistry, + ConqueryConfig config) { super(form, user, submittedDataset, storage, datasetRegistry, config); } @@ -84,11 +90,12 @@ protected void setAdditionalStatusFields(Subject subject, FullExecutionStatus st } if (subQueries.size() != 1) { // The sub-query size might also be zero if the backend just delegates the form further to another backend. Forms with more subqueries are not yet supported - log.trace("Column description is not generated for {} ({} from Form {}), because the form does not consits of a single subquery. Subquery size was {}.", - subQueries.size(), - this.getClass().getSimpleName(), - getId(), - getSubmitted().getClass().getSimpleName() + log.trace( + "Column description is not generated for {} ({} from Form {}), because the form does not consits of a single subquery. Subquery size was {}.", + subQueries.size(), + this.getClass().getSimpleName(), + getId(), + getSubmitted().getClass().getSimpleName() ); return; } @@ -111,23 +118,36 @@ public void doInitExecutable() { // Create hard ref to couple the object lifetime of the subqueries to this object initializedSubQueryHardRef = subQueries; - this.subQueries = subQueries.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getId())); + this.subQueries = subQueries.entrySet() + .stream() + .collect( + Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getId())); } @NotNull private Map resolveOrCreateSubExecutions() { if (subQueries != null && !subQueries.isEmpty()) { // This execution was already executed once, to we resolve the corresponding subqueries - return subQueries.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> (ManagedQuery) e.getValue().resolve())); + return subQueries.entrySet() + .stream() + .collect( + Collectors.toMap(Map.Entry::getKey, e -> (ManagedQuery) e.getValue().resolve())); } return getSubmitted().createSubQueries() - .entrySet() - .stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - e -> e.getValue().toManagedExecution(getOwner(), getDataset(), getMetaStorage(), getDatasetRegistry(), getConfig()) - )); + .entrySet() + .stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + e -> e.getValue() + .toManagedExecution( + getOwner(), + getDataset(), + getMetaStorage(), + getDatasetRegistry(), + getConfig()) + )); } @Override @@ -138,7 +158,9 @@ public void start() { @Override public List generateColumnDescriptions(boolean isInitialized, ConqueryConfig config) { - return ((ManagedQuery) subQueries.values().iterator().next().resolve()).generateColumnDescriptions(isInitialized, config); + return ((ManagedQuery) subQueries.values().iterator().next().resolve()).generateColumnDescriptions( + isInitialized, + config); } @Override @@ -153,7 +175,9 @@ public List collectResultInfos() { @Override @JsonIgnore public List getResultInfos() { - ExecutionManager.InternalExecutionInfo executionInfo = getNamespace().getExecutionManager().getExecutionInfo(getId()); + ExecutionManager.InternalExecutionInfo executionInfo = getNamespace().getExecutionManager() + .getExecutionInfo( + getId()); return executionInfo.getResultInfos(); } @@ -177,19 +201,29 @@ public OptionalLong resultRowCount() { public boolean allSubQueriesDone() { synchronized (this) { - return subQueries.values().stream().map(ManagedExecutionId::resolve).allMatch(q -> q.getState().equals(ExecutionState.DONE)); + return subQueries.values() + .stream() + .map(ManagedExecutionId::resolve) + .allMatch( + q -> q.getState().equals(ExecutionState.DONE)); } } protected Map resolvedSubQueries() { - return subQueries.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> (ManagedQuery) e.getValue().resolve())); + return subQueries.entrySet() + .stream() + .collect( + Collectors.toMap(Map.Entry::getKey, e -> (ManagedQuery) e.getValue().resolve())); } public ExecuteForm createExecutionMessage() { Preconditions.checkState(isInitialized(), "Was not initialized"); - return new ExecuteForm(getId(), initializedSubQueryHardRef.values() - .stream() - .collect(Collectors.toMap(ManagedExecution::getId, ManagedQuery::getQuery)) + return new ExecuteForm( + getId(), + initializedSubQueryHardRef.values() + .stream() + .collect( + Collectors.toMap(ManagedExecution::getId, ManagedQuery::getQuery)) ); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQuery.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQuery.java index e6976e9d751..4646938161d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQuery.java @@ -1,12 +1,12 @@ package com.bakdata.conquery.models.forms.managed; +import jakarta.validation.Valid; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotNull; import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.function.Consumer; -import jakarta.validation.Valid; -import jakarta.validation.constraints.Min; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.apiv1.forms.IndexPlacement; import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; @@ -60,14 +60,12 @@ public void resolve(QueryResolveContext context) { query.resolve(context.withDateAggregationMode(DateAggregationMode.MERGE)); features.resolve(context.withDateAggregationMode(DateAggregationMode.NONE)); - boolean noDates = Visitable.stream(query) - .noneMatch(v -> - switch (v) { - case CQConcept cqConcept -> cqConcept.isAggregateEventDates(); - case CQExternal external -> external.containsDates(); - default -> false; - } - ); + boolean noDates = Visitable.stream(query).noneMatch(v -> switch (v) { + case CQConcept cqConcept -> cqConcept.isAggregateEventDates(); + case CQExternal external -> external.containsDates(); + default -> false; + } + ); if (noDates) { throw new ConqueryError.RelativeFormMissingDatesError(); @@ -78,10 +76,15 @@ public void resolve(QueryResolveContext context) { @Override public RelativeFormQueryPlan createQueryPlan(QueryPlanContext context) { return new RelativeFormQueryPlan( - query.createQueryPlan(context), - // At the moment we do not use the dates of feature and outcome query - features.createQueryPlan(context), - indexSelector, indexPlacement, timeCountBefore, timeCountAfter, timeUnit, resolutionsAndAlignmentMap + query.createQueryPlan(context), + // At the moment we do not use the dates of feature and outcome query + features.createQueryPlan(context), + indexSelector, + indexPlacement, + timeCountBefore, + timeCountAfter, + timeUnit, + resolutionsAndAlignmentMap ); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java index a1da6bcc8a1..0b895007205 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java @@ -28,6 +28,7 @@ import lombok.RequiredArgsConstructor; import lombok.ToString; import lombok.extern.slf4j.Slf4j; +import org.jspecify.annotations.NonNull; @Slf4j @Getter @@ -82,7 +83,10 @@ public Optional execute(QueryExecutionContext ctx, Entity int size = calculateCompleteLength(); EntityResult contained = preResult.get(); // Gather all validity dates from prerequisite - CDateSet dateSet = query.getValidityDateAggregator().map(Aggregator::createAggregationResult).orElseGet(CDateSet::createEmpty); + CDateSet dateSet = query.getValidityDateAggregator() + .map(Aggregator::createAggregationResult) + .orElseGet( + CDateSet::createEmpty); final OptionalInt sampled = indexSelector.sample(dateSet); @@ -92,14 +96,22 @@ public Optional execute(QueryExecutionContext ctx, Entity List results = new ArrayList<>(); results.add(new Object[size]); return Optional.of( - ResultModifier.modify(new MultilineEntityResult(entity.getId(), results), ResultModifier.existAggValuesSetterFor(getAggregators(), OptionalInt - .of(getFirstAggregatorPosition()))) + ResultModifier.modify( + new MultilineEntityResult(entity.getId(), results), + ResultModifier.existAggValuesSetterFor( + getAggregators(), + OptionalInt.of(getFirstAggregatorPosition()))) ); } int sample = sampled.getAsInt(); - List contexts = - DateContext.generateRelativeContexts(sample, indexPlacement, timeCountBefore, timeCountAfter, timeUnit, resolutionsAndAlignmentMap); + List contexts = DateContext.generateRelativeContexts( + sample, + indexPlacement, + timeCountBefore, + timeCountAfter, + timeUnit, + resolutionsAndAlignmentMap); // create feature and outcome plans featureSubquery = createSubQuery(featurePlan, contexts, FeatureGroup.FEATURE); @@ -147,11 +159,18 @@ private void assertResultWidth(EntityResult subResult, int resultWidth) { return; } - throw new IllegalStateException(String.format("Aggregator number (%d) and result number (%d) are not the same", resultWidth, resultColumnCount)); + throw new IllegalStateException( + String.format( + "Aggregator number (%d) and result number (%d) are not the same", + resultWidth, + resultColumnCount)); } - private static FormQueryPlan createSubQuery(ArrayConceptQueryPlan subPlan, List contexts, FeatureGroup featureGroup) { + private static FormQueryPlan createSubQuery( + ArrayConceptQueryPlan subPlan, + List contexts, + FeatureGroup featureGroup) { List list = new ArrayList<>(contexts); list.removeIf(dctx -> dctx.getFeatureGroup() != featureGroup); @@ -168,7 +187,7 @@ public boolean isOfInterest(Entity entity) { } @Override - public Optional> getValidityDateAggregator() { + public @NonNull Optional> getValidityDateAggregator() { DateAggregator agg = new DateAggregator(DateAggregationAction.MERGE); featureSubquery.getValidityDateAggregator().ifPresent(agg::register); diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/util/Alignment.java b/backend/src/main/java/com/bakdata/conquery/models/forms/util/Alignment.java index 97ede3ad296..86500f5fc30 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/util/Alignment.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/util/Alignment.java @@ -1,15 +1,15 @@ package com.bakdata.conquery.models.forms.util; -import com.bakdata.conquery.models.common.daterange.CDateRange; -import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.Getter; -import lombok.RequiredArgsConstructor; - import java.util.List; import java.util.Map; import java.util.OptionalInt; import java.util.function.Function; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + /** * Specifier for the alignment of {@link DateContext}s of a certain resolution. * The alignment provides a method to sub divide a dateRangeMask into ranges aligned to the calendar equivalent. @@ -18,7 +18,7 @@ */ @RequiredArgsConstructor public enum Alignment { - NO_ALIGN(List::of){ + NO_ALIGN(List::of) { @Override protected Map getAmountPerResolution() { return Map.of(Resolution.COMPLETE, 1); @@ -28,17 +28,22 @@ protected Map getAmountPerResolution() { @Override protected Map getAmountPerResolution() { return Map.of( - Resolution.YEARS, 365, - Resolution.QUARTERS, 90, - Resolution.DAYS, 1); + Resolution.YEARS, + 365, + Resolution.QUARTERS, + 90, + Resolution.DAYS, + 1); } }, QUARTER(CDateRange::getCoveredQuarters) { @Override protected Map getAmountPerResolution() { return Map.of( - Resolution.YEARS, 4, - Resolution.QUARTERS, 1); + Resolution.YEARS, + 4, + Resolution.QUARTERS, + 1); } }, YEAR(CDateRange::getCoveredYears) { @@ -67,4 +72,3 @@ public OptionalInt getAmountForResolution(Resolution resolution) { return OptionalInt.empty(); } } - diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/util/AlignmentReference.java b/backend/src/main/java/com/bakdata/conquery/models/forms/util/AlignmentReference.java index 0cb22624c6a..10f6766c6e3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/util/AlignmentReference.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/util/AlignmentReference.java @@ -1,10 +1,10 @@ package com.bakdata.conquery.models.forms.util; +import java.util.List; + import com.bakdata.conquery.models.common.daterange.CDateRange; import com.google.common.collect.Lists; -import java.util.List; - /** * Specifies whether the sub date ranges (which have the maximum length specified by the {@link Resolution}) * are aligned with regard to beginning or the end of a date range mask. This affects the generated sub date ranges diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/util/CalendarUnit.java b/backend/src/main/java/com/bakdata/conquery/models/forms/util/CalendarUnit.java index dea1471b289..ce2a801a842 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/util/CalendarUnit.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/util/CalendarUnit.java @@ -10,9 +10,7 @@ */ @RequiredArgsConstructor public enum CalendarUnit { - DAYS(Alignment.DAY), - QUARTERS(Alignment.QUARTER), - YEARS(Alignment.YEAR); + DAYS(Alignment.DAY), QUARTERS(Alignment.QUARTER), YEARS(Alignment.YEAR); @Getter private final Alignment alignment; diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/util/DateContext.java b/backend/src/main/java/com/bakdata/conquery/models/forms/util/DateContext.java index e79f6eb93c6..14f1eed35be 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/util/DateContext.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/util/DateContext.java @@ -4,7 +4,6 @@ import java.time.temporal.IsoFields; import java.util.*; import java.util.function.Function; - import javax.annotation.Nullable; import com.bakdata.conquery.apiv1.forms.FeatureGroup; @@ -45,13 +44,14 @@ public class DateContext { */ @Getter private Integer index = null; - + /** * The date from which the relative context were generated. */ - @Getter @Setter + @Getter + @Setter private LocalDate eventDate = null; - + /** * Indicates under which temporal subdivision mode this instance was created. */ @@ -70,18 +70,27 @@ public class DateContext { * @param resolutionAndAlignment The resolutions to produce and their alignment * @return A sorted list of all generated contexts */ - public static List generateAbsoluteContexts(CDateRange dateRangeMask, List resolutionAndAlignment) { + public static List generateAbsoluteContexts( + CDateRange dateRangeMask, + List resolutionAndAlignment) { List dcList = new ArrayList<>(); for (ExportForm.ResolutionAndAlignment mode : resolutionAndAlignment) { - Function> divider = getDateRangeSubdivider(AlignmentReference.START, mode.getResolution(), mode.getAlignment()); + Function> divider = getDateRangeSubdivider( + AlignmentReference.START, + mode.getResolution(), + mode.getAlignment()); // Start counting index form 0 for every subdivision mode int index = 0; for (CDateRange quarterInMask : divider.apply(dateRangeMask)) { index++; - DateContext dc = new DateContext(quarterInMask, FeatureGroup.SINGLE_GROUP, + DateContext dc = new DateContext( + quarterInMask, + FeatureGroup.SINGLE_GROUP, // For now there is no index for complete - mode.getResolution().equals(Resolution.COMPLETE) ? null : index, null, mode.getResolution()); + mode.getResolution().equals(Resolution.COMPLETE) ? null : index, + null, + mode.getResolution()); dcList.add(dc); } } @@ -92,8 +101,13 @@ public static List generateAbsoluteContexts(CDateRange dateRangeMas * Factory function that produces a list of {@link CDateRange}s from a given dateRangeMask according to the given * {@link AlignmentReference}, {@link Resolution} and {@link Alignment}. */ - public static Function> getDateRangeSubdivider(AlignmentReference alignRef, Resolution resolution, Alignment alignment){ - int alignedPerResolution = resolution.getAmountForAlignment(alignment).orElseThrow(() -> new ConqueryError.ExecutionCreationPlanDateContextError(alignment, resolution)); + public static Function> getDateRangeSubdivider( + AlignmentReference alignRef, + Resolution resolution, + Alignment alignment) { + int alignedPerResolution = resolution.getAmountForAlignment(alignment) + .orElseThrow( + () -> new ConqueryError.ExecutionCreationPlanDateContextError(alignment, resolution)); if (alignedPerResolution == 1) { // When the alignment fits the resolution we can use the alignment subdivision directly @@ -101,9 +115,10 @@ public static Function> getDateRangeSubdivider(Align } return (dateRange) -> { - List alignedSubdivisions = alignRef.getAlignedIterationDirection(alignment.getSubdivider().apply(dateRange)); + List alignedSubdivisions = alignRef.getAlignedIterationDirection( + alignment.getSubdivider().apply(dateRange)); - if(alignedSubdivisions.isEmpty()){ + if (alignedSubdivisions.isEmpty()) { return alignedSubdivisions; } @@ -140,10 +155,16 @@ public static Function> getDateRangeSubdivider(Align * The event (a certain day) itself is expanded to a date range according to the desired alignment and the indexPlacement * determines to which group it belongs. */ - public static List generateRelativeContexts(int event, IndexPlacement indexPlacement, int featureTime, int outcomeTime, CalendarUnit timeUnit, List resolutionAndAlignment) { + public static List generateRelativeContexts( + int event, + IndexPlacement indexPlacement, + int featureTime, + int outcomeTime, + CalendarUnit timeUnit, + List resolutionAndAlignment) { if (featureTime < 1 && outcomeTime < 1) { - throw new IllegalArgumentException("Both relative times were smaller than 1 (featureTime: " + featureTime - + "; outcomeTime: " + outcomeTime + ")"); + throw new IllegalArgumentException( + "Both relative times were smaller than 1 (featureTime: " + featureTime + "; outcomeTime: " + outcomeTime + ")"); } List dcList = new ArrayList<>(); @@ -153,11 +174,17 @@ public static List generateRelativeContexts(int event, IndexPlaceme CDateRange outcomeRange = generateOutcomeRange(event, indexPlacement, outcomeTime, timeUnit); - for(ExportForm.ResolutionAndAlignment mode : resolutionAndAlignment) { - Function> featureRangeDivider = getDateRangeSubdivider(AlignmentReference.END, mode.getResolution(), mode.getAlignment()); - Function> outcomeRangeDivider = getDateRangeSubdivider(AlignmentReference.START, mode.getResolution(), mode.getAlignment()); + for (ExportForm.ResolutionAndAlignment mode : resolutionAndAlignment) { + Function> featureRangeDivider = getDateRangeSubdivider( + AlignmentReference.END, + mode.getResolution(), + mode.getAlignment()); + Function> outcomeRangeDivider = getDateRangeSubdivider( + AlignmentReference.START, + mode.getResolution(), + mode.getAlignment()); - if(featureRange != null) { + if (featureRange != null) { List featureRanges = featureRangeDivider.apply(featureRange); /* @@ -169,12 +196,12 @@ public static List generateRelativeContexts(int event, IndexPlaceme int index = featureRanges.size(); for (CDateRange subRange : featureRanges) { DateContext dc = new DateContext( - subRange, - FeatureGroup.FEATURE, - // For now there is no index for complete - mode.getResolution().equals(Resolution.COMPLETE) ? null : -index, - eventdate, - mode.getResolution() + subRange, + FeatureGroup.FEATURE, + // For now there is no index for complete + mode.getResolution().equals(Resolution.COMPLETE) ? null : -index, + eventdate, + mode.getResolution() ); index--; dcList.add(dc); @@ -186,12 +213,12 @@ public static List generateRelativeContexts(int event, IndexPlaceme int index = 1; for (CDateRange subRange : outcomeRangeDivider.apply(outcomeRange)) { DateContext dc = new DateContext( - subRange, - FeatureGroup.OUTCOME, - // For now there is no index for complete - mode.getResolution().equals(Resolution.COMPLETE) ? null : index, - eventdate, - mode.getResolution() + subRange, + FeatureGroup.OUTCOME, + // For now there is no index for complete + mode.getResolution().equals(Resolution.COMPLETE) ? null : index, + eventdate, + mode.getResolution() ); index++; dcList.add(dc); @@ -203,7 +230,7 @@ public static List generateRelativeContexts(int event, IndexPlaceme /** * Calculates the feature range. - * + * * @param event The event date to which the range is relative. * @param indexPlacement Indicates to which {@link FeatureGroup} the event index * belongs. @@ -213,8 +240,12 @@ public static List generateRelativeContexts(int event, IndexPlaceme * * TODO Replace this and the following function by a function with an interface like this: private static CDateRange generateRelativeRange(int event, boolean indexIncluded, int relativeAmount, CalendarUnit timeUnit) */ - private static CDateRange generateFeatureRange(int event, IndexPlacement indexPlacement, int featureTime, CalendarUnit timeUnit) { - if(featureTime <= 0){ + private static CDateRange generateFeatureRange( + int event, + IndexPlacement indexPlacement, + int featureTime, + CalendarUnit timeUnit) { + if (featureTime <= 0) { return null; } if (indexPlacement.equals(IndexPlacement.BEFORE)) { @@ -222,8 +253,8 @@ private static CDateRange generateFeatureRange(int event, IndexPlacement indexPl case DAYS: return CDateRange.of(event - featureTime + 1, event); case QUARTERS: - LocalDate eventRangeStart = QuarterUtils - .getFirstDayOfQuarter(CDate.toLocalDate(event).minus(featureTime - 1, IsoFields.QUARTER_YEARS)); + LocalDate eventRangeStart = QuarterUtils.getFirstDayOfQuarter( + CDate.toLocalDate(event).minus(featureTime - 1, IsoFields.QUARTER_YEARS)); LocalDate eventRangeEnd = QuarterUtils.getLastDayOfQuarter(event); return CDateRange.of(eventRangeStart, eventRangeEnd); default: @@ -235,10 +266,10 @@ private static CDateRange generateFeatureRange(int event, IndexPlacement indexPl case DAYS: return CDateRange.of(event - featureTime, event - 1); case QUARTERS: - LocalDate eventRangeStart = QuarterUtils - .getFirstDayOfQuarter(CDate.toLocalDate(event).minus(featureTime, IsoFields.QUARTER_YEARS)); - LocalDate eventRangeEnd = QuarterUtils - .getLastDayOfQuarter(CDate.toLocalDate(event).minus(1, IsoFields.QUARTER_YEARS)); + LocalDate eventRangeStart = QuarterUtils.getFirstDayOfQuarter( + CDate.toLocalDate(event).minus(featureTime, IsoFields.QUARTER_YEARS)); + LocalDate eventRangeEnd = QuarterUtils.getLastDayOfQuarter( + CDate.toLocalDate(event).minus(1, IsoFields.QUARTER_YEARS)); return CDateRange.of(eventRangeStart, eventRangeEnd); default: throw new IllegalArgumentException("Unsupported Resolution: " + timeUnit); @@ -247,7 +278,7 @@ private static CDateRange generateFeatureRange(int event, IndexPlacement indexPl /** * Calculates the outcome range. - * + * * @param event The event date to which the range is relative. * @param indexPlacement Indicates to which {@link FeatureGroup} the event index * belongs. @@ -255,11 +286,15 @@ private static CDateRange generateFeatureRange(int event, IndexPlacement indexPl * @param resolution The time unit. * @return The outcome range. */ - private static CDateRange generateOutcomeRange(int event, IndexPlacement indexPlacement, int outcomeTime, CalendarUnit resolution) { + private static CDateRange generateOutcomeRange( + int event, + IndexPlacement indexPlacement, + int outcomeTime, + CalendarUnit resolution) { if (outcomeTime <= 0) { return null; } - switch(indexPlacement) { + switch (indexPlacement) { case AFTER: switch (resolution) { case DAYS: @@ -280,10 +315,10 @@ private static CDateRange generateOutcomeRange(int event, IndexPlacement indexPl case DAYS: return CDateRange.of(event + 1, event + outcomeTime); case QUARTERS: - LocalDate eventRangeStart = QuarterUtils - .getFirstDayOfQuarter(CDate.toLocalDate(event).plus(1, IsoFields.QUARTER_YEARS)); - LocalDate eventRangeEnd = QuarterUtils - .getLastDayOfQuarter(CDate.toLocalDate(event).plus(outcomeTime, IsoFields.QUARTER_YEARS)); + LocalDate eventRangeStart = QuarterUtils.getFirstDayOfQuarter( + CDate.toLocalDate(event).plus(1, IsoFields.QUARTER_YEARS)); + LocalDate eventRangeEnd = QuarterUtils.getLastDayOfQuarter( + CDate.toLocalDate(event).plus(outcomeTime, IsoFields.QUARTER_YEARS)); return CDateRange.of(eventRangeStart, eventRangeEnd); default: throw new IllegalArgumentException("Unsupported Resolution: " + resolution); @@ -292,8 +327,8 @@ private static CDateRange generateOutcomeRange(int event, IndexPlacement indexPl default: throw new IllegalArgumentException("Unsupported index placement: " + indexPlacement); - + } - + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/util/ResultModifier.java b/backend/src/main/java/com/bakdata/conquery/models/forms/util/ResultModifier.java index acd47df56d9..22899044854 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/util/ResultModifier.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/util/ResultModifier.java @@ -16,7 +16,7 @@ public class ResultModifier { /** * Modifies the given result lines with the given function. - * + * * If the result is not contained this method creates a default result row * anyways. */ @@ -27,11 +27,16 @@ public static T modify(T inResult, UnaryOperator existAggValuesSetterFor(List> aggregators, OptionalInt firstAggPos) { + public static ChainableUnaryOperator existAggValuesSetterFor( + List> aggregators, + OptionalInt firstAggPos) { return (result) -> setExistAggValues(aggregators, result, firstAggPos); } - private static Object[] setExistAggValues(List> aggregators, Object[] result, OptionalInt firstAggPos) { + private static Object[] setExistAggValues( + List> aggregators, + Object[] result, + OptionalInt firstAggPos) { int aggIdx = firstAggPos.orElse(0); /* * Special handling here, because a subquery might not be contained but has an diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/IdMap.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/IdMap.java index 2ede6ce4d76..8f7170903d6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/IdMap.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/IdMap.java @@ -1,104 +1,106 @@ -package com.bakdata.conquery.models.identifiable; - -import java.util.Collection; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.stream.Stream; -import jakarta.validation.Valid; - -import com.bakdata.conquery.models.identifiable.ids.Id; -import com.google.common.collect.ForwardingMap; - -/** - * A typesafe mapping for the ID-system that does not allow a remapping of an existing key. - * - * @implNote implementation of {@link Iterable} is dropped, because hibernate could not decide on how to validate this map (either with an map-extractor or an iterable-extractor). - */ -public class IdMap> extends ForwardingMap { - - @Valid - private final ConcurrentMap map; - - public IdMap() { - map = new ConcurrentHashMap(); - } - - public IdMap(Collection collection) { - map = new ConcurrentHashMap<>(); - for(V value : collection) { - map.put(value.getId(), value); - } - } - - @Override - public Set> entrySet(){ - return map.entrySet(); - } - - @Override - public Collection values() { - return map.values(); - } - - public Stream stream() { - return map.values().stream(); - } - - @Override - public int size() { - return map.size(); - } - - public V getOrFail(ID id) { - V res = map.get(id); - if(res==null) { - throw new NoSuchElementException("Could not find an element called '"+id+"'"); - } - return res; - } - - private void addToMap(V entry) { - // The following cast should be unnecessary, but intellij is complaining without it. Please leave it here for now - V old = (V) map.put(entry.getId(), entry); - if(old != null && !old.equals(entry)) { - throw new IllegalStateException("The element "+entry.getId()+" is present twice in this map."); - } - } - - public Optional getOptional(ID id) { - return Optional.ofNullable(map.get(id)); - } - - public boolean add(V entry) { - addToMap(entry); - return true; - } - - public V update(V entry) { - return map.put(entry.getId(), entry); - } - - public V remove(ID id) { - V obj = map.remove(id); - return obj; - } - - @Override @Deprecated - public V remove(Object object) { - return super.remove(object); - } - - @Override @Deprecated - public V put(ID key, V value) { - return super.put(key, value); - } - - @Override - protected Map delegate() { - return map; - } -} +package com.bakdata.conquery.models.identifiable; + +import jakarta.validation.Valid; +import java.util.Collection; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.stream.Stream; + +import com.bakdata.conquery.models.identifiable.ids.Id; +import com.google.common.collect.ForwardingMap; + +/** + * A typesafe mapping for the ID-system that does not allow a remapping of an existing key. + * + * @implNote implementation of {@link Iterable} is dropped, because hibernate could not decide on how to validate this map (either with an map-extractor or an iterable-extractor). + */ +public class IdMap> extends ForwardingMap { + + @Valid + private final ConcurrentMap map; + + public IdMap() { + map = new ConcurrentHashMap(); + } + + public IdMap(Collection collection) { + map = new ConcurrentHashMap<>(); + for (V value : collection) { + map.put(value.getId(), value); + } + } + + @Override + public Set> entrySet() { + return map.entrySet(); + } + + @Override + public Collection values() { + return map.values(); + } + + public Stream stream() { + return map.values().stream(); + } + + @Override + public int size() { + return map.size(); + } + + public V getOrFail(ID id) { + V res = map.get(id); + if (res == null) { + throw new NoSuchElementException("Could not find an element called '" + id + "'"); + } + return res; + } + + private void addToMap(V entry) { + // The following cast should be unnecessary, but intellij is complaining without it. Please leave it here for now + V old = (V) map.put(entry.getId(), entry); + if (old != null && !old.equals(entry)) { + throw new IllegalStateException("The element " + entry.getId() + " is present twice in this map."); + } + } + + public Optional getOptional(ID id) { + return Optional.ofNullable(map.get(id)); + } + + public boolean add(V entry) { + addToMap(entry); + return true; + } + + public V update(V entry) { + return map.put(entry.getId(), entry); + } + + public V remove(ID id) { + V obj = map.remove(id); + return obj; + } + + @Override + @Deprecated + public V remove(Object object) { + return super.remove(object); + } + + @Override + @Deprecated + public V put(ID key, V value) { + return super.put(key, value); + } + + @Override + protected Map delegate() { + return map; + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/IdResolvingException.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/IdResolvingException.java index 9737116f2cb..b454ebcdf5a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/IdResolvingException.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/IdResolvingException.java @@ -12,12 +12,12 @@ public class IdResolvingException extends ConqueryError { public IdResolvingException(Id id) { this.id = id; - } + } public IdResolvingException(Id id, Throwable cause) { - super(ConqueryError.asConqueryError(cause)); + super(ConqueryError.asConqueryError(cause)); this.id = id; - } + } @Override public String getMessageTemplate(ErrorMessages errorMessages) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/IdentifiableImpl.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/IdentifiableImpl.java index c7f9eed5883..76ddd51a45c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/IdentifiableImpl.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/IdentifiableImpl.java @@ -52,8 +52,7 @@ public boolean equals(Object obj) { IdentifiableImpl other = (IdentifiableImpl) obj; if (getId() == null) { return other.getId() == null; - } - else { + } else { return getId().equals(other.getId()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/LabeledNamespaceIdentifiable.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/LabeledNamespaceIdentifiable.java index 8549b203dd2..cb5f9bbe396 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/LabeledNamespaceIdentifiable.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/LabeledNamespaceIdentifiable.java @@ -11,13 +11,13 @@ import lombok.ToString; public abstract class LabeledNamespaceIdentifiable> extends NamespacedIdentifiable { - private static final CharMatcher DEFAULT_NAME_UNWANTED = CharMatcher.is(IdUtil.JOIN_CHAR).or(CharMatcher.whitespace()); + private static final CharMatcher DEFAULT_NAME_UNWANTED = CharMatcher.is(IdUtil.JOIN_CHAR) + .or( + CharMatcher.whitespace()); /** - * shown in the frontend - * - * @jsonExample "someLabel" + * Shown in the frontend */ private String label; diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/NamespacedStorageProvider.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/NamespacedStorageProvider.java index 4653c17a17d..e89183e8c36 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/NamespacedStorageProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/NamespacedStorageProvider.java @@ -16,8 +16,10 @@ public interface NamespacedStorageProvider extends Injectable { static NamespacedStorageProvider getInjected(DeserializationContext ctxt) throws JsonMappingException { - return (NamespacedStorageProvider) ctxt - .findInjectableValue(NamespacedStorageProvider.class.getName(), null, null); + return (NamespacedStorageProvider) ctxt.findInjectableValue( + NamespacedStorageProvider.class.getName(), + null, + null); } /** diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/Id.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/Id.java index 3195a705d5f..6418bb5d5a9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/Id.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/Id.java @@ -64,11 +64,9 @@ public final TYPE resolve() { throw newIdResolveException(); } return o; - } - catch (IdResolvingException e) { + } catch (IdResolvingException e) { throw e; - } - catch (Exception e) { + } catch (Exception e) { throw newIdResolveException(e); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/IdIterator.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/IdIterator.java index aef16a9386c..f91e57f7f06 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/IdIterator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/IdIterator.java @@ -6,30 +6,31 @@ import lombok.Getter; import lombok.ToString; -@Getter @ToString +@Getter +@ToString @EqualsAndHashCode(onlyExplicitlyIncluded = true) //TODO is this class necessary? public class IdIterator { @EqualsAndHashCode.Include private final List data; private int index; - + public IdIterator(List data) { this.data = data; this.index = data.size(); } - + public List getRemaining() { - if(index <= 0) { - throw new IllegalStateException("No more remaining values in "+data); + if (index <= 0) { + throw new IllegalStateException("No more remaining values in " + data); } return data.subList(0, index); } - + public int remaining() { return index; } - + public String next() { return data.get(--index); } @@ -43,12 +44,12 @@ public void consumeAll() { } public IdIterator splitOff(int n) { - IdIterator result = new IdIterator(data.subList(index-n, index)); + IdIterator result = new IdIterator(data.subList(index - n, index)); index -= n; return result; } public void internNext() { - data.set(index-1, data.get(index-1).intern()); + data.set(index - 1, data.get(index - 1).intern()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/IdUtil.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/IdUtil.java index d92b075b4e4..1e7df5e94d7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/IdUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/IdUtil.java @@ -40,7 +40,8 @@ public final class IdUtil { try { Class returnType = MethodUtils.getAccessibleMethod(cl, methodName).getReturnType(); if (!Id.class.isAssignableFrom(returnType) || Id.class.equals(returnType)) { - throw new IllegalStateException("The type `" + returnType + "` of `" + cl + "#" + methodName + "` is not a specific subtype of Id"); + throw new IllegalStateException( + "The type `" + returnType + "` of `" + cl + "#" + methodName + "` is not a specific subtype of Id"); } if (NamespacedIdentifiable.class.isAssignableFrom(cl) != NamespacedId.class.isAssignableFrom(returnType)) { @@ -51,8 +52,7 @@ public final class IdUtil { CLASS_TO_ID_MAP.put(cl, result); return (Class) Objects.requireNonNull(result); - } - catch (SecurityException e1) { + } catch (SecurityException e1) { throw new IllegalStateException("The type " + cl + " has no " + methodName + " method", e1); } } @@ -95,13 +95,13 @@ default ID createId(List parts) { default ID checkNoRemaining(ID id, IdIterator remaining, List allParts) { if (remaining.remaining() > 0) { throw new IllegalStateException( - String.format( - "After parsing '%s' as id '%s' of type %s there are parts remaining: '%s'", - IdUtil.JOINER.join(allParts), - id, - id.getClass().getSimpleName(), - IdUtil.JOINER.join(remaining.getRemaining()) - ) + String.format( + "After parsing '%s' as id '%s' of type %s there are parts remaining: '%s'", + IdUtil.JOINER.join(allParts), + id, + id.getClass().getSimpleName(), + IdUtil.JOINER.join(remaining.getRemaining()) + ) ); } return id; @@ -133,8 +133,7 @@ static List asComponents(String dataset, String id) { if (split.length > 0 && split[0].equals(dataset)) { //if already prefixed result = split; - } - else { + } else { result = new String[split.length + 1]; result[0] = dataset; System.arraycopy(split, 0, result, 1, split.length); @@ -143,4 +142,4 @@ static List asComponents(String dataset, String id) { } } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/NamespacedId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/NamespacedId.java index a913d6f8534..5b7cbbb168a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/NamespacedId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/NamespacedId.java @@ -41,4 +41,4 @@ public void setDomain(NamespacedStorageProvider provider) { getDataset().setDomain(provider); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/BucketId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/BucketId.java index 08b89f05c0c..c2f760f9a70 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/BucketId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/BucketId.java @@ -27,8 +27,7 @@ public DatasetId getDataset() { @Override public Bucket get() { - return assertWorkerStorage(getDomain().getStorage(getDataset())) - .getBucket(this); + return assertWorkerStorage(getDomain().getStorage(getDataset())).getBucket(this); } @Override @@ -54,4 +53,4 @@ public BucketId parseInternally(IdIterator parts) { return new BucketId(parent, bucket); } } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/CBlockId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/CBlockId.java index 1b0b01377f2..c5ba2368bb4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/CBlockId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/CBlockId.java @@ -27,8 +27,7 @@ public DatasetId getDataset() { @Override public CBlock get() { - return assertWorkerStorage(getDomain().getStorage(getDataset())) - .getCBlock(this); + return assertWorkerStorage(getDomain().getStorage(getDataset())).getCBlock(this); } @Override @@ -54,4 +53,4 @@ public CBlockId parseInternally(IdIterator parts) { return new CBlockId(block, connector); } } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ColumnId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ColumnId.java index 83ca534fd3e..5480e84f009 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ColumnId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ColumnId.java @@ -27,9 +27,7 @@ public DatasetId getDataset() { @Override public Column get() { - return getDomain().getStorage(getDataset()) - .getTable(getTable()) - .getColumnByName(getColumn()); + return getDomain().getStorage(getDataset()).getTable(getTable()).getColumnByName(getColumn()); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ConceptId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ConceptId.java index 75ef8ec5df9..2cd388857a7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ConceptId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ConceptId.java @@ -27,8 +27,7 @@ public final class ConceptId extends ConceptElementId> implements Aut @Override public Concept get() { - return getDomain().getStorage(getDataset()) - .getConcept(this); + return getDomain().getStorage(getDataset()).getConcept(this); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ConceptSelectId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ConceptSelectId.java index e416ad036bf..188823292a1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ConceptSelectId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ConceptSelectId.java @@ -10,11 +10,12 @@ import lombok.EqualsAndHashCode; import lombok.Getter; -@Getter @EqualsAndHashCode(callSuper=true) +@Getter +@EqualsAndHashCode(callSuper = true) public final class ConceptSelectId extends SelectId { private final ConceptId concept; - + public ConceptSelectId(ConceptId concept, String select) { super(select); this.concept = concept; @@ -33,8 +34,7 @@ public DatasetId getDataset() { @Override public UniversalSelect get() { - return (UniversalSelect) getDomain().getStorage(getDataset()) - .getConcept(concept).getSelectByName(getSelect()); + return (UniversalSelect) getDomain().getStorage(getDataset()).getConcept(concept).getSelectByName(getSelect()); } @@ -59,4 +59,4 @@ public ConceptSelectId parseInternally(IdIterator parts) { return new ConceptSelectId(parent, name); } } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ConceptTreeChildId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ConceptTreeChildId.java index ba30b0c0bc6..19b621efb5f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ConceptTreeChildId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ConceptTreeChildId.java @@ -12,12 +12,14 @@ import lombok.EqualsAndHashCode; import lombok.Getter; -@Getter @AllArgsConstructor @EqualsAndHashCode(callSuper=false) +@Getter +@AllArgsConstructor +@EqualsAndHashCode(callSuper = false) public final class ConceptTreeChildId extends ConceptElementId { private final ConceptElementId parent; private final String name; - + @Override public DatasetId getDataset() { return parent.getDataset(); @@ -25,8 +27,7 @@ public DatasetId getDataset() { @Override public ConceptTreeChild get() { - Concept concept = getDomain().getStorage(getDataset()) - .getConcept(findConcept()); + Concept concept = getDomain().getStorage(getDataset()).getConcept(findConcept()); if (concept == null) { return null; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ConnectorSelectId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ConnectorSelectId.java index 22da7f5dbc9..fcd72930dd7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ConnectorSelectId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ConnectorSelectId.java @@ -10,11 +10,12 @@ import lombok.EqualsAndHashCode; import lombok.Getter; -@Getter @EqualsAndHashCode(callSuper=true) +@Getter +@EqualsAndHashCode(callSuper = true) public final class ConnectorSelectId extends SelectId { private final ConnectorId connector; - + public ConnectorSelectId(ConnectorId connector, String select) { super(select); this.connector = connector; @@ -34,9 +35,10 @@ public DatasetId getDataset() { @Override public Select get() { return getDomain().getStorage(getDataset()) - .getConcept(findConcept()) - .getConnectorByName(getConnector().getConnector()) - .getSelectByName(getSelect()); + .getConcept(findConcept()) + .getConnectorByName( + getConnector().getConnector()) + .getSelectByName(getSelect()); } @Override @@ -61,4 +63,4 @@ public ConnectorSelectId parseInternally(IdIterator parts) { return new ConnectorSelectId(parent, name); } } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/FilterId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/FilterId.java index 1328235f2c4..59c2eaf2927 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/FilterId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/FilterId.java @@ -28,8 +28,10 @@ public DatasetId getDataset() { @Override public Filter get() { return getDomain().getStorage(getDataset()) - .getConcept(connector.getConcept()).getConnectorByName(connector.getConnector()) - .getFilterByName(getFilter()); + .getConcept(connector.getConcept()) + .getConnectorByName( + connector.getConnector()) + .getFilterByName(getFilter()); } @Override @@ -55,4 +57,4 @@ public FilterId parseInternally(IdIterator parts) { return new FilterId(parent, filter); } } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/GroupId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/GroupId.java index 0e76639e83c..a67583da4d2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/GroupId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/GroupId.java @@ -1,50 +1,50 @@ -package com.bakdata.conquery.models.identifiable.ids.specific; - -import java.util.Collection; -import java.util.List; - -import com.bakdata.conquery.models.auth.entities.Group; -import com.bakdata.conquery.models.identifiable.ids.Id; -import com.bakdata.conquery.models.identifiable.ids.IdIterator; -import com.bakdata.conquery.models.identifiable.ids.IdUtil; -import lombok.EqualsAndHashCode; -import lombok.Getter; - -@Getter -@EqualsAndHashCode(callSuper=false) -public final class GroupId extends PermissionOwnerId { - - public static final String TYPE = "group"; - - private final String group; - - public GroupId(String group) { - super(); - this.group = group; - } - - public void collectComponents(List components) { - components.add(TYPE); - components.add(group); - } - - @Override - public void collectIds(Collection> collect) { - collect.add(this); - } - - @Override - public Group get() { - return getDomain().getGroup(this); - } - - public enum Parser implements IdUtil.Parser { - INSTANCE; - - @Override - public GroupId parseInternally(IdIterator parts) { - return (GroupId) PermissionOwnerId.Parser.INSTANCE.parse(parts); - } - } - -} +package com.bakdata.conquery.models.identifiable.ids.specific; + +import java.util.Collection; +import java.util.List; + +import com.bakdata.conquery.models.auth.entities.Group; +import com.bakdata.conquery.models.identifiable.ids.Id; +import com.bakdata.conquery.models.identifiable.ids.IdIterator; +import com.bakdata.conquery.models.identifiable.ids.IdUtil; +import lombok.EqualsAndHashCode; +import lombok.Getter; + +@Getter +@EqualsAndHashCode(callSuper = false) +public final class GroupId extends PermissionOwnerId { + + public static final String TYPE = "group"; + + private final String group; + + public GroupId(String group) { + super(); + this.group = group; + } + + public void collectComponents(List components) { + components.add(TYPE); + components.add(group); + } + + @Override + public void collectIds(Collection> collect) { + collect.add(this); + } + + @Override + public Group get() { + return getDomain().getGroup(this); + } + + public enum Parser implements IdUtil.Parser { + INSTANCE; + + @Override + public GroupId parseInternally(IdIterator parts) { + return (GroupId) PermissionOwnerId.Parser.INSTANCE.parse(parts); + } + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ImportColumnId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ImportColumnId.java index b4eaad77a32..8d153440e3e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ImportColumnId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ImportColumnId.java @@ -53,4 +53,4 @@ public ImportColumnId parseInternally(IdIterator parts) { return new ImportColumnId(parent, column); } } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ImportId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ImportId.java index d7e8a947cc1..5bc9e36ee71 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ImportId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ImportId.java @@ -53,4 +53,4 @@ public ImportId parseInternally(IdIterator parts) { return new ImportId(parent, tag); } } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/PermissionOwnerId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/PermissionOwnerId.java index dae03162f49..b349440f986 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/PermissionOwnerId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/PermissionOwnerId.java @@ -1,34 +1,34 @@ -package com.bakdata.conquery.models.identifiable.ids.specific; - -import com.bakdata.conquery.models.auth.entities.PermissionOwner; -import com.bakdata.conquery.models.identifiable.ids.IdIterator; -import com.bakdata.conquery.models.identifiable.ids.IdUtil; -import com.bakdata.conquery.models.identifiable.ids.MetaId; -import lombok.EqualsAndHashCode; -import lombok.RequiredArgsConstructor; - - -@RequiredArgsConstructor -@EqualsAndHashCode(callSuper = false) -public abstract sealed class PermissionOwnerId> extends MetaId - permits UserId, RoleId, GroupId { - - - public enum Parser implements IdUtil.Parser> { - INSTANCE; - - @Override - public PermissionOwnerId parseInternally(IdIterator parts) { - final String ownerId = parts.next(); - final String type = parts.next(); - - return switch (type) { - case UserId.TYPE -> new UserId(ownerId); - case RoleId.TYPE -> new RoleId(ownerId); - case GroupId.TYPE -> new GroupId(ownerId); - default -> throw new IllegalStateException("Unknown permission owner type: " + type); - }; - } - } - -} +package com.bakdata.conquery.models.identifiable.ids.specific; + +import com.bakdata.conquery.models.auth.entities.PermissionOwner; +import com.bakdata.conquery.models.identifiable.ids.IdIterator; +import com.bakdata.conquery.models.identifiable.ids.IdUtil; +import com.bakdata.conquery.models.identifiable.ids.MetaId; +import lombok.EqualsAndHashCode; +import lombok.RequiredArgsConstructor; + + +@RequiredArgsConstructor +@EqualsAndHashCode(callSuper = false) +public abstract sealed class PermissionOwnerId> extends MetaId + permits UserId, RoleId, GroupId { + + + public enum Parser implements IdUtil.Parser> { + INSTANCE; + + @Override + public PermissionOwnerId parseInternally(IdIterator parts) { + final String ownerId = parts.next(); + final String type = parts.next(); + + return switch (type) { + case UserId.TYPE -> new UserId(ownerId); + case RoleId.TYPE -> new RoleId(ownerId); + case GroupId.TYPE -> new GroupId(ownerId); + default -> throw new IllegalStateException("Unknown permission owner type: " + type); + }; + } + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/RoleId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/RoleId.java index df56a577272..93fe3c0f8e2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/RoleId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/RoleId.java @@ -1,50 +1,50 @@ -package com.bakdata.conquery.models.identifiable.ids.specific; - -import java.util.Collection; -import java.util.List; - -import com.bakdata.conquery.models.auth.entities.Role; -import com.bakdata.conquery.models.identifiable.ids.Id; -import com.bakdata.conquery.models.identifiable.ids.IdIterator; -import com.bakdata.conquery.models.identifiable.ids.IdUtil; -import lombok.EqualsAndHashCode; -import lombok.Getter; - - -@Getter -@EqualsAndHashCode(callSuper=false) -public final class RoleId extends PermissionOwnerId { - public static final String TYPE = "role"; - - private final String role; - - public RoleId(String mandator) { - super(); - this.role = mandator; - } - - public void collectComponents(List components) { - components.add(TYPE); - components.add(role); - } - - @Override - public void collectIds(Collection> collect) { - collect.add(this); - } - - @Override - public Role get() { - return getDomain().getRole(this); - } - - enum Parser implements IdUtil.Parser { - INSTANCE; - - @Override - public RoleId parseInternally(IdIterator parts) { - return (RoleId) PermissionOwnerId.Parser.INSTANCE.parse(parts); - } - } - -} +package com.bakdata.conquery.models.identifiable.ids.specific; + +import java.util.Collection; +import java.util.List; + +import com.bakdata.conquery.models.auth.entities.Role; +import com.bakdata.conquery.models.identifiable.ids.Id; +import com.bakdata.conquery.models.identifiable.ids.IdIterator; +import com.bakdata.conquery.models.identifiable.ids.IdUtil; +import lombok.EqualsAndHashCode; +import lombok.Getter; + + +@Getter +@EqualsAndHashCode(callSuper = false) +public final class RoleId extends PermissionOwnerId { + public static final String TYPE = "role"; + + private final String role; + + public RoleId(String mandator) { + super(); + this.role = mandator; + } + + public void collectComponents(List components) { + components.add(TYPE); + components.add(role); + } + + @Override + public void collectIds(Collection> collect) { + collect.add(this); + } + + @Override + public Role get() { + return getDomain().getRole(this); + } + + enum Parser implements IdUtil.Parser { + INSTANCE; + + @Override + public RoleId parseInternally(IdIterator parts) { + return (RoleId) PermissionOwnerId.Parser.INSTANCE.parse(parts); + } + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/SelectId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/SelectId.java index e1d7e5b8bbb..b77292b9b02 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/SelectId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/SelectId.java @@ -37,4 +37,4 @@ public SelectId parseInternally(IdIterator parts) { return ConnectorSelectId.Parser.INSTANCE.parse(parts); } } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/StructureNodeId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/StructureNodeId.java index 04ad83ba43c..e341089f713 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/StructureNodeId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/StructureNodeId.java @@ -31,8 +31,7 @@ public StructureNode get() { public void collectComponents(List components) { if (parent != null) { parent.collectComponents(components); - } - else { + } else { dataset.collectComponents(components); } components.add(structureNode); @@ -43,8 +42,7 @@ public void collectIds(Collection> collect) { collect.add(this); if (parent != null) { parent.collectIds(collect); - } - else { + } else { dataset.collectIds(collect); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/UserId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/UserId.java index 177c59e7c58..3e5ce7cd96f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/UserId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/UserId.java @@ -1,50 +1,50 @@ -package com.bakdata.conquery.models.identifiable.ids.specific; - -import java.util.Collection; -import java.util.List; - -import com.bakdata.conquery.models.auth.entities.User; -import com.bakdata.conquery.models.identifiable.ids.Id; -import com.bakdata.conquery.models.identifiable.ids.IdIterator; -import com.bakdata.conquery.models.identifiable.ids.IdUtil; -import lombok.EqualsAndHashCode; -import lombok.Getter; - -@EqualsAndHashCode(callSuper=false) -public final class UserId extends PermissionOwnerId { - public static final String TYPE = "user"; - - @Getter - private final String name; - - public UserId(String name) { - super(); - this.name = name; - } - - @Override - public void collectComponents(List components) { - components.add(TYPE); - components.add(name); - } - - @Override - public void collectIds(Collection> collect) { - collect.add(this); - } - - @Override - public User get() { - return getDomain().getUser(this); - } - - public enum Parser implements IdUtil.Parser { - INSTANCE; - - @Override - public UserId parseInternally(IdIterator parts) { - return (UserId) PermissionOwnerId.Parser.INSTANCE.parse(parts); - } - } - -} +package com.bakdata.conquery.models.identifiable.ids.specific; + +import java.util.Collection; +import java.util.List; + +import com.bakdata.conquery.models.auth.entities.User; +import com.bakdata.conquery.models.identifiable.ids.Id; +import com.bakdata.conquery.models.identifiable.ids.IdIterator; +import com.bakdata.conquery.models.identifiable.ids.IdUtil; +import lombok.EqualsAndHashCode; +import lombok.Getter; + +@EqualsAndHashCode(callSuper = false) +public final class UserId extends PermissionOwnerId { + public static final String TYPE = "user"; + + @Getter + private final String name; + + public UserId(String name) { + super(); + this.name = name; + } + + @Override + public void collectComponents(List components) { + components.add(TYPE); + components.add(name); + } + + @Override + public void collectIds(Collection> collect) { + collect.add(this); + } + + @Override + public User get() { + return getDomain().getUser(this); + } + + public enum Parser implements IdUtil.Parser { + INSTANCE; + + @Override + public UserId parseInternally(IdIterator parts) { + return (UserId) PermissionOwnerId.Parser.INSTANCE.parse(parts); + } + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ValidityDateId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ValidityDateId.java index 3dfd1c7b8ac..f7f745b0a9d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ValidityDateId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ValidityDateId.java @@ -22,9 +22,10 @@ public class ValidityDateId extends NamespacedId { @Override public ValidityDate get() { return getDomain().getStorage(getDataset()) - .getConcept(getConnector().getConcept()) - .getConnectorByName(getConnector().getConnector()) - .getValidityDateByName(getValidityDate()); + .getConcept(getConnector().getConcept()) + .getConnectorByName( + getConnector().getConnector()) + .getValidityDateByName(getValidityDate()); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java index aa8253c1c96..b064a1473b4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java @@ -58,7 +58,10 @@ public class EntityIdMap { /** * Read incoming CSV-file extracting Id-Mappings for {@link ExternalId} and {@link EntityPrintId}. */ - public static EntityIdMap generateIdMapping(CsvParser parser, List mappers, NamespaceStorage namespaceStorage) { + public static EntityIdMap generateIdMapping( + CsvParser parser, + List mappers, + NamespaceStorage namespaceStorage) { EntityIdMap mapping = new EntityIdMap(namespaceStorage); diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityPrintId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityPrintId.java index 7d780e8ff11..640d79203c5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityPrintId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityPrintId.java @@ -1,9 +1,9 @@ package com.bakdata.conquery.models.identifiable.mapping; +import jakarta.validation.constraints.NotNull; import java.util.Arrays; import com.fasterxml.jackson.annotation.JsonCreator; -import jakarta.validation.constraints.NotNull; import lombok.AccessLevel; import lombok.Data; import lombok.RequiredArgsConstructor; diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/PrintIdMapper.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/PrintIdMapper.java index 3f010e8b238..969292a4b43 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/PrintIdMapper.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/PrintIdMapper.java @@ -7,5 +7,5 @@ * Maps an internalId to an external representation. */ public interface PrintIdMapper { - EntityPrintId map(EntityResult entityResult); + EntityPrintId map(EntityResult entityResult); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java b/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java index 22967bd0992..fefa8357de6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java @@ -1,5 +1,9 @@ package com.bakdata.conquery.models.index; +import java.util.Collection; +import java.util.List; +import java.util.Map; + import com.bakdata.conquery.apiv1.FilterTemplate; import com.bakdata.conquery.apiv1.frontend.FrontendValue; import com.bakdata.conquery.util.search.Search; @@ -9,10 +13,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.time.StopWatch; -import java.util.Collection; -import java.util.List; -import java.util.Map; - @Slf4j @ToString public class FrontendValueIndex implements Index { @@ -39,9 +39,9 @@ public FrontendValueIndex(Search delegate, String valueTemplate, @Override public void put(String internalValue, Map templateToConcrete) { final FrontendValue feValue = new FrontendValue( - internalValue, - templateToConcrete.get(valueTemplate), - templateToConcrete.get(optionValueTemplate) + internalValue, + templateToConcrete.get(valueTemplate), + templateToConcrete.get(optionValueTemplate) ); delegate.addItem(feValue, SearchProcessor.extractKeywords(feValue)); diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndexKey.java b/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndexKey.java index 0065e886ad6..eaa67c0bfbd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndexKey.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndexKey.java @@ -35,7 +35,12 @@ public class FrontendValueIndexKey implements IndexKey { private final String internalColumn; - public FrontendValueIndexKey(URI csv, String internalColumn, String valueTemplate, String optionValueTemplate, Supplier> indexSupplier) { + public FrontendValueIndexKey( + URI csv, + String internalColumn, + String valueTemplate, + String optionValueTemplate, + Supplier> indexSupplier) { this.indexSupplier = indexSupplier; this.valueTemplate = valueTemplate; diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java b/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java index b06f89aba31..3f703211585 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java @@ -44,63 +44,69 @@ public class IndexService implements Injectable { * The Index interface offers methods, that are only relevant for the {@link InternToExternMapper} like {@link Index#external(String)}. * We should clean this up and split the async CSV parsing from the index functionality. */ - private final LoadingCache> mappings = CacheBuilder.newBuilder().recordStats().build(new CacheLoader<>() { - @NotNull - @Override - public Index load(@NotNull IndexKey key) throws Exception { - - final StopWatch timer = StopWatch.createStarted(); - - log.info("Started to parse mapping {}", key); - - final Map emptyDefaults = computeEmptyDefaults(key); - - final Index int2ext = key.createIndex(); - - final CsvParser csvParser = new CsvParser(csvParserSettings); - - try (InputStream inputStream = key.getCsv().toURL().openStream()) { - - final IterableResult records = csvParser.iterateRecords(inputStream); - - // Set default to "" for all columns - final String[] headers = records.getContext().headers(); - records.getContext().recordMetaData().setDefaultValueOfColumns("", headers); - - // Iterate records - for (Record row : records) { - // There can be multiple templates and multiple external values, hence the right side is a map - final Pair> pair = computeInternalExternal(key, csvParser, row); - if (pair == null) { - continue; + private final LoadingCache> mappings = CacheBuilder.newBuilder() + .recordStats() + .build( + new CacheLoader<>() { + @NotNull + @Override + public Index load(@NotNull IndexKey key) throws Exception { + + final StopWatch timer = StopWatch.createStarted(); + + log.info("Started to parse mapping {}", key); + + final Map emptyDefaults = computeEmptyDefaults(key); + + final Index int2ext = key.createIndex(); + + final CsvParser csvParser = new CsvParser(csvParserSettings); + + try (InputStream inputStream = key.getCsv().toURL().openStream()) { + + final IterableResult records = csvParser.iterateRecords(inputStream); + + // Set default to "" for all columns + final String[] headers = records.getContext().headers(); + records.getContext().recordMetaData().setDefaultValueOfColumns("", headers); + + // Iterate records + for (Record row : records) { + // There can be multiple templates and multiple external values, hence the right side is a map + final Pair> pair = computeInternalExternal(key, csvParser, row); + if (pair == null) { + continue; + } + + final String internalValue = pair.getLeft(); + final Map externalValue = pair.getRight(); + + // If the computed value is equal to a template without any values, replace it with value + externalValue.replaceAll( + (template, value) -> emptyDefaults.get(template).equals(value) ? internalValue : value); + + try { + int2ext.put(internalValue, externalValue); + } catch (IllegalArgumentException e) { + log.warn( + "Skipping mapping '{}'->'{}' in row {}, because there was already a mapping", + internalValue, + externalValue, + csvParser.getContext().currentLine(), + LogUtil.passExceptionOnTrace(log, e) + ); + } + } } - final String internalValue = pair.getLeft(); - final Map externalValue = pair.getRight(); + // Run finalizing operations on the index + int2ext.finalizer(); - // If the computed value is equal to a template without any values, replace it with value - externalValue.replaceAll((template, value) -> emptyDefaults.get(template).equals(value) ? internalValue : value); + log.info("Finished parsing mapping {} with {} entries, within {}", key, int2ext.size(), timer); - try { - int2ext.put(internalValue, externalValue); - } - catch (IllegalArgumentException e) { - log.warn("Skipping mapping '{}'->'{}' in row {}, because there was already a mapping", - internalValue, externalValue, csvParser.getContext().currentLine(), - LogUtil.passExceptionOnTrace(log,e) - ); - } + return int2ext; } - } - - // Run finalizing operations on the index - int2ext.finalizer(); - - log.info("Finished parsing mapping {} with {} entries, within {}", key, int2ext.size(), timer); - - return int2ext; - } - }); + }); public IndexService(CsvParserSettings csvParserSettings) { this.csvParserSettings = csvParserSettings.clone(); @@ -108,16 +114,23 @@ public IndexService(CsvParserSettings csvParserSettings) { } @Nullable - private Pair> computeInternalExternal(@NotNull IndexKey key, CsvParser csvParser, Record row) { - final StringSubstitutor substitutor = new StringSubstitutor(row::getString, "{{", "}}", StringSubstitutor.DEFAULT_ESCAPE); + private Pair> computeInternalExternal( + @NotNull IndexKey key, + CsvParser csvParser, + Record row) { + final StringSubstitutor substitutor = new StringSubstitutor( + row::getString, + "{{", + "}}", + StringSubstitutor.DEFAULT_ESCAPE); final String internalValue = row.getString(key.getInternalColumn()); if (internalValue == null) { - log.trace("Could not create a mapping for row {} because the cell for the internal value was empty. Row: {}", csvParser.getContext().currentLine(), - log.isTraceEnabled() - ? StringUtils.join(row.toFieldMap()) - : null + log.trace( + "Could not create a mapping for row {} because the cell for the internal value was empty. Row: {}", + csvParser.getContext().currentLine(), + log.isTraceEnabled() ? StringUtils.join(row.toFieldMap()) : null ); return null; } @@ -134,12 +147,19 @@ private Map computeTemplates(StringSubstitutor substitutor, List final CharMatcher whitespaceMatcher = CharMatcher.whitespace(); return externalTemplates.stream() - .distinct() - .collect(Collectors.toMap(Functions.identity(), value -> whitespaceMatcher.trimAndCollapseFrom(substitutor.replace(value), ' '))); + .distinct() + .collect( + Collectors.toMap( + Functions.identity(), + value -> whitespaceMatcher.trimAndCollapseFrom(substitutor.replace(value), ' '))); } private Map computeEmptyDefaults(IndexKey key) { - final StringSubstitutor substitutor = new StringSubstitutor((ignored) -> "", "{{", "}}", StringSubstitutor.DEFAULT_ESCAPE); + final StringSubstitutor substitutor = new StringSubstitutor( + (ignored) -> "", + "{{", + "}}", + StringSubstitutor.DEFAULT_ESCAPE); final List externalTemplates = key.getExternalTemplates(); @@ -164,8 +184,7 @@ public void evictCache() { public > I getIndex(@NotNull IndexKey key) throws IndexCreationException { try { return (I) mappings.get(key); - } - catch (ExecutionException e) { + } catch (ExecutionException e) { throw new IndexCreationException(key, e); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/InternToExternMapper.java b/backend/src/main/java/com/bakdata/conquery/models/index/InternToExternMapper.java index 7132dfd72f6..27e13e3e850 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/InternToExternMapper.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/InternToExternMapper.java @@ -54,7 +54,8 @@ public InternToExternMapperId createId() { public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printSettings) { if (isAllowMultiple()) { - return new OneToManyMappingPrinter(this).andThen(printerFactory.getListPrinter(printerFactory.getStringPrinter(printSettings), printSettings)); + return new OneToManyMappingPrinter(this).andThen( + printerFactory.getListPrinter(printerFactory.getStringPrinter(printSettings), printSettings)); } return new OneToOneMappingPrinter(this, printerFactory.getStringPrinter(printSettings)); diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/MapIndexKey.java b/backend/src/main/java/com/bakdata/conquery/models/index/MapIndexKey.java index 33b7a7b12cc..8ab769cb2b6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/MapIndexKey.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/MapIndexKey.java @@ -14,7 +14,6 @@ public class MapIndexKey implements IndexKey { private final boolean allowMultiple; - @Override public List getExternalTemplates() { return List.of(externalTemplate); @@ -22,7 +21,7 @@ public List getExternalTemplates() { @Override public Index createIndex() { - if (allowMultiple){ + if (allowMultiple) { return new MapMultiIndex(externalTemplate); } return new MapIndex(externalTemplate); diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java b/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java index b6d11044137..6167bf4d57e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java @@ -2,13 +2,13 @@ import static com.bakdata.conquery.util.io.LogUtil.passExceptionOnTrace; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; import java.net.URI; import java.util.Collection; import java.util.Collections; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.jackson.Initializing; @@ -76,7 +76,12 @@ public class MapInternToExternMapper extends InternToExternMapper implements Ini @EqualsAndHashCode.Exclude private CompletableFuture> int2ext; - public MapInternToExternMapper(@NotEmpty String name, @NotNull URI csv, @NotEmpty String internalColumn, @NotEmpty String externalTemplate, boolean allowMultiple) { + public MapInternToExternMapper( + @NotEmpty String name, + @NotNull URI csv, + @NotEmpty String internalColumn, + @NotEmpty String externalTemplate, + boolean allowMultiple) { this.name = name; this.csv = csv; this.internalColumn = internalColumn; @@ -101,13 +106,12 @@ public synchronized void init() { int2ext = CompletableFuture.supplyAsync(() -> { try { return mapIndex.>getIndex(key); - } - catch (IndexCreationException e) { + } catch (IndexCreationException e) { throw new IllegalStateException(e); } }).whenComplete((m, e) -> { if (e != null) { - log.warn("Unable to get index: {} (enable TRACE for exception)", key, passExceptionOnTrace(log,e)); + log.warn("Unable to get index: {} (enable TRACE for exception)", key, passExceptionOnTrace(log, e)); } }); } @@ -123,10 +127,12 @@ public Collection externalMultiple(String internalValue) { } return mapped; - } - catch (InterruptedException | ExecutionException e) { + } catch (InterruptedException | ExecutionException e) { // Should never be reached - log.warn("Unable to resolve mapping for internal value {} (enable TRACE for exception)", internalValue, passExceptionOnTrace(log,e)); + log.warn( + "Unable to resolve mapping for internal value {} (enable TRACE for exception)", + internalValue, + passExceptionOnTrace(log, e)); } } @@ -156,10 +162,12 @@ public String external(String internalValue) { if (indexAvailable()) { try { return int2ext.get().external(internalValue); - } - catch (InterruptedException | ExecutionException e) { + } catch (InterruptedException | ExecutionException e) { // Should never be reached - log.warn("Unable to resolve mapping for internal value {} (enable TRACE for exception)", internalValue, passExceptionOnTrace(log,e)); + log.warn( + "Unable to resolve mapping for internal value {} (enable TRACE for exception)", + internalValue, + passExceptionOnTrace(log, e)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/MapMultiIndex.java b/backend/src/main/java/com/bakdata/conquery/models/index/MapMultiIndex.java index 35b01bfff42..4f534db7830 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/MapMultiIndex.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/MapMultiIndex.java @@ -18,8 +18,7 @@ public class MapMultiIndex implements Index { @Override public void put(String key, Map templateToConcrete) { - delegate.computeIfAbsent(key, (ignored) -> new HashSet<>()) - .add(templateToConcrete.get(externalTemplate)); + delegate.computeIfAbsent(key, (ignored) -> new HashSet<>()).add(templateToConcrete.get(externalTemplate)); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/CalculateCBlocksJob.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/CalculateCBlocksJob.java index 5725965d79c..70e1924c897 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/jobs/CalculateCBlocksJob.java +++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/CalculateCBlocksJob.java @@ -56,11 +56,12 @@ public void execute() throws Exception { final ListeningExecutorService executorService = MoreExecutors.listeningDecorator(getExecutorService()); - final List> futures = - tasks.stream() - .map(executorService::submit) - .peek(fut -> fut.addListener(this::incrementProgressReporter, MoreExecutors.directExecutor())) - .collect(Collectors.toList()); + final List> futures = tasks.stream() + .map(executorService::submit) + .peek( + fut -> fut.addListener(this::incrementProgressReporter, MoreExecutors.directExecutor())) + .collect( + Collectors.toList()); final ListenableFuture all = Futures.allAsList(futures); @@ -68,8 +69,7 @@ public void execute() throws Exception { while (!all.isDone()) { try { all.get(1, TimeUnit.MINUTES); - } - catch (TimeoutException exception) { + } catch (TimeoutException exception) { log.debug("submitted={}, pool={}", tasks.size(), getExecutorService()); if (log.isTraceEnabled() && getExecutorService() instanceof ThreadPoolExecutor) { @@ -110,7 +110,9 @@ private static class CalculationInformationProcessor implements Runnable { public void run() { try { if (bucketManager.hasCBlock(getCBlockId())) { - log.trace("Skipping calculation of CBlock[{}] because its already present in the BucketManager.", getCBlockId()); + log.trace( + "Skipping calculation of CBlock[{}] because its already present in the BucketManager.", + getCBlockId()); return; } @@ -122,8 +124,7 @@ public void run() { bucketManager.addCalculatedCBlock(cBlock); storage.addCBlock(cBlock); - } - catch (Exception e) { + } catch (Exception e) { throw new RuntimeException("Exception in CalculateCBlocksJob %s".formatted(getCBlockId()), e); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/Job.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/Job.java index 1dc87a64e4b..b2d47fed373 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/jobs/Job.java +++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/Job.java @@ -28,10 +28,12 @@ public boolean isCancelled() { } public abstract void execute() throws Exception; + public abstract String getLabel(); @Override public String toString() { - return this.getClass().getSimpleName()+"(label='"+getLabel()+"' progress="+progressReporter.getEstimate()+")"; + return this.getClass().getSimpleName() + "(label='" + getLabel() + "' progress=" + progressReporter + .getEstimate() + ")"; } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/JobExecutor.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/JobExecutor.java index 0a7c9924f61..62e65b5d81a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/jobs/JobExecutor.java +++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/JobExecutor.java @@ -32,7 +32,7 @@ public JobExecutor(String name, boolean failOnError) { } public void add(Job job) { - if(closed.get()) { + if (closed.get()) { log.warn("Tried to add a job to a closed JobManager: {}", job.getLabel()); return; } @@ -50,7 +50,7 @@ public boolean cancelJob(UUID jobId) { final Job job = currentJob.get(); - if(job != null && job.getJobId().equals(jobId)){ + if (job != null && job.getJobId().equals(jobId)) { job.cancel(); return true; } @@ -67,18 +67,18 @@ public List getJobs() { jobs.addAll(this.jobs); return jobs; } - + /** * Checks if the executor is currently working on a job or if there are jobs left in its queue. * If so, the executor is busy. * @return True if there is work left to do for this executor */ public boolean isBusy() { - if(currentJob.get() != null) { + if (currentJob.get() != null) { log.trace("JobExecutor {} is still working on a task.", getName()); return true; } - if(!jobs.isEmpty()) { + if (!jobs.isEmpty()) { log.trace("JobExecutor {} has still work in the queue.", getName()); return true; } @@ -97,10 +97,10 @@ public void close() { public void run() { ConqueryMDC.setLocation(this.getName()); - while(!closed.get()) { + while (!closed.get()) { Job job; try { - while((job =jobs.poll(100, TimeUnit.MILLISECONDS))!=null) { + while ((job = jobs.poll(100, TimeUnit.MILLISECONDS)) != null) { currentJob.set(job); job.getProgressReporter().start(); Stopwatch timer = Stopwatch.createStarted(); @@ -108,7 +108,7 @@ public void run() { final Timer.Context time = JobMetrics.getJobExecutorTimer(job); try { - if(job.isCancelled()){ + if (job.isCancelled()) { log.trace("{} skipping cancelled job {}", this.getName(), job); continue; } @@ -118,8 +118,7 @@ public void run() { job.execute(); ConqueryMDC.setLocation(this.getName()); - } - catch (Throwable e) { + } catch (Throwable e) { ConqueryMDC.setLocation(this.getName()); log.error("Job {} failed", job, e); diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManager.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManager.java index a7a055b841c..6fa089cbdf2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManager.java +++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManager.java @@ -40,9 +40,14 @@ public void addFastJob(Job job) { public List getJobStatus() { return getSlowJobs().stream() - .map(job -> new JobStatus(job.getJobId(), job.getProgressReporter().getProgress(), job.getLabel(), job.isCancelled())) - .sorted() - .collect(Collectors.toList()); + .map( + job -> new JobStatus( + job.getJobId(), + job.getProgressReporter().getProgress(), + job.getLabel(), + job.isCancelled())) + .sorted() + .collect(Collectors.toList()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManagerStatus.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManagerStatus.java index 4611c2720e5..64159606a4b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManagerStatus.java +++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManagerStatus.java @@ -1,11 +1,11 @@ package com.bakdata.conquery.models.jobs; +import jakarta.validation.constraints.NotNull; import java.time.Duration; import java.time.LocalDateTime; import java.util.Collections; import java.util.List; import javax.annotation.Nullable; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/JobStatus.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/JobStatus.java index 1184a4298dc..fa49b8fef59 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/jobs/JobStatus.java +++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/JobStatus.java @@ -25,9 +25,6 @@ public class JobStatus implements Comparable { @Override public int compareTo(@NotNull JobStatus o) { - return BY_PROGRESS - .thenComparing(BY_LABEL) - .thenComparing(BY_UUID) - .compare(this, o); + return BY_PROGRESS.thenComparing(BY_LABEL).thenComparing(BY_UUID).compare(this, o); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/ReactingJob.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/ReactingJob.java index f4388782ef3..da2f46fe829 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/jobs/ReactingJob.java +++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/ReactingJob.java @@ -6,24 +6,25 @@ import lombok.AllArgsConstructor; import lombok.ToString; -@AllArgsConstructor @ToString +@AllArgsConstructor +@ToString public class ReactingJob, CTX extends NetworkMessageContext> extends Job { private MESSAGE message; @ToString.Exclude private CTX context; - + @Override public void execute() { try { message.react(context); - } catch(Exception e) { - throw new RuntimeException("Failed while processing the message "+message, e); + } catch (Exception e) { + throw new RuntimeException("Failed while processing the message " + message, e); } } @Override public String getLabel() { - return "reacting to "+message; + return "reacting to " + message; } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/SimpleJob.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/SimpleJob.java index 868cd8bc292..3ce2459f7df 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/jobs/SimpleJob.java +++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/SimpleJob.java @@ -13,7 +13,7 @@ public class SimpleJob extends Job { public void execute() throws Exception { runner.execute(); } - + public interface Executable { void execute() throws Exception; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java index c4b297ffa7b..ee21f122f0e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java +++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java @@ -49,29 +49,35 @@ public void execute() throws Exception { log.info("BEGIN loading SourceSearch"); // collect all SelectFilters to create searches for them - final List> allSelectFilters = - getAllSelectFilters(storage); + final List> allSelectFilters = getAllSelectFilters(storage); // Unfortunately the is no ClassToInstanceMultimap yet - final Map, Set> collectedSearchables = - allSelectFilters.stream() - .map(SelectFilter::getSearchReferences) - .flatMap(Collection::stream) - // Group Searchables into "Columns" and other "Searchables" - .collect(Collectors.groupingBy(s -> s instanceof Column ? Column.class : Searchable.class, Collectors.toSet())); + final Map, Set> collectedSearchables = allSelectFilters.stream() + .map( + SelectFilter::getSearchReferences) + .flatMap(Collection::stream) + // Group Searchables into "Columns" and other "Searchables" + .collect( + Collectors.groupingBy( + s -> s instanceof Column ? Column.class : Searchable.class, + Collectors.toSet())); log.debug("Found {} searchable Objects.", collectedSearchables.values().stream().mapToLong(Set::size).sum()); - Set managerSearchables = collectedSearchables.getOrDefault(Searchable.class, Collections.emptySet()); + Set managerSearchables = collectedSearchables.getOrDefault( + Searchable.class, + Collections.emptySet()); searchProcessor.indexManagerResidingSearches(managerSearchables, getCancelledState(), getProgressReporter()); // The following cast is safe - final Set searchableColumns = (Set) collectedSearchables.getOrDefault(Column.class, Collections.emptySet()); + final Set searchableColumns = (Set) collectedSearchables.getOrDefault( + Column.class, + Collections.emptySet()); log.debug("Start collecting column values: {}", Arrays.toString(searchableColumns.toArray())); registerColumnValuesInSearch.accept(searchableColumns); @@ -80,16 +86,16 @@ public void execute() throws Exception { } - @NotNull public static List> getAllSelectFilters(NamespaceStorage storage) { - try(Stream> allConcepts = storage.getAllConcepts()) { - return allConcepts - .flatMap(c -> c.getConnectors().stream()) - .flatMap(co -> co.collectAllFilters().stream()) - .filter(SelectFilter.class::isInstance) - .map(f -> ((SelectFilter) f)) - .collect(Collectors.toList()); + try (Stream> allConcepts = storage.getAllConcepts()) { + return allConcepts.flatMap(c -> c.getConnectors().stream()) + .flatMap( + co -> co.collectAllFilters().stream()) + .filter(SelectFilter.class::isInstance) + .map( + f -> ((SelectFilter) f)) + .collect(Collectors.toList()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/SlowMessage.java b/backend/src/main/java/com/bakdata/conquery/models/messages/SlowMessage.java index ded0eb151b0..7661072b076 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/SlowMessage.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/SlowMessage.java @@ -7,5 +7,6 @@ public interface SlowMessage extends Message { @JsonIgnore ProgressReporter getProgressReporter(); + void setProgressReporter(ProgressReporter progressReporter); -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/NamespaceMessage.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/NamespaceMessage.java index b308a38c035..fc28927b524 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/NamespaceMessage.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/NamespaceMessage.java @@ -7,9 +7,12 @@ import lombok.Getter; import lombok.Setter; -@Getter @Setter +@Getter +@Setter public abstract class NamespaceMessage extends NamespacedMessage implements SlowMessage { - @JsonIgnore @Getter @Setter + @JsonIgnore + @Getter + @Setter private ProgressReporter progressReporter; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/WorkerMessage.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/WorkerMessage.java index 837ce15ac1f..0694428ed89 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/WorkerMessage.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/WorkerMessage.java @@ -11,8 +11,11 @@ * WorkerMessages are always slow to ensure that they are processed in order and that there are no conflict * when ids are resolved. */ -@Getter @Setter -public abstract class WorkerMessage extends NamespacedMessage implements SlowMessage { - @JsonIgnore @Getter @Setter +@Getter +@Setter +public abstract class WorkerMessage extends NamespacedMessage implements SlowMessage { + @JsonIgnore + @Getter + @Setter private ProgressReporter progressReporter; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/AddImport.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/AddImport.java index 659f2a24933..d6893963add 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/AddImport.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/AddImport.java @@ -10,7 +10,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -@CPSType(id="ADD_IMPORT", base=NamespacedMessage.class) +@CPSType(id = "ADD_IMPORT", base = NamespacedMessage.class) @RequiredArgsConstructor(onConstructor_ = @JsonCreator) @Slf4j @Getter diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/CollectColumnValuesJob.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/CollectColumnValuesJob.java index e69de29bb2d..8b137891791 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/CollectColumnValuesJob.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/CollectColumnValuesJob.java @@ -0,0 +1 @@ + diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/CollectColumnValuesMessage.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/CollectColumnValuesMessage.java index e49894e76cd..e42d76303db 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/CollectColumnValuesMessage.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/CollectColumnValuesMessage.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.messages.namespaces.specific; +import jakarta.validation.constraints.NotNull; import java.util.Arrays; import java.util.HashSet; import java.util.List; @@ -10,7 +11,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.events.stores.root.StringStore; @@ -71,69 +71,84 @@ public class CollectColumnValuesJob extends Job { public void execute() throws Exception { final Map> table2Buckets; try (Stream allBuckets = context.getStorage().getAllBucketIds()) { - table2Buckets = allBuckets - .collect(Collectors.groupingBy(bucketId -> bucketId.getImp().getTable())); + table2Buckets = allBuckets.collect(Collectors.groupingBy(bucketId -> bucketId.getImp().getTable())); } - final BasicThreadFactory threadFactory = - new BasicThreadFactory.Builder() - .namingPattern(getClass().getSimpleName() + "-%s".formatted(context.getInfo().getName()) + "-Worker-%d").build(); + final BasicThreadFactory threadFactory = new BasicThreadFactory.Builder().namingPattern( + getClass().getSimpleName() + "-%s".formatted(context.getInfo().getName()) + "-Worker-%d").build(); - final ListeningExecutorService jobsExecutorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(MAX_THREADS, threadFactory)); + final ListeningExecutorService jobsExecutorService = MoreExecutors.listeningDecorator( + Executors.newFixedThreadPool(MAX_THREADS, threadFactory)); final AtomicInteger done = new AtomicInteger(); - getProgressReporter().setMax(columns.stream() - .filter(column -> table2Buckets.get(column.getTable()) != null).count()); + getProgressReporter().setMax( + columns.stream().filter(column -> table2Buckets.get(column.getTable()) != null).count()); log.info("BEGIN collecting values from {}", Arrays.toString(columns.toArray())); - final List> futures = - columns.stream() - .filter(column -> table2Buckets.get(column.getTable()) != null) - .map(ColumnId::resolve) - .map(column -> jobsExecutorService.submit(() -> { - final List buckets = table2Buckets.get(column.getTable().getId()); - - // Keep track of the values we have already seen - HashSet seenValues = new HashSet<>(); - - final Stream values = buckets.stream() - .map(BucketId::resolve) - .flatMap(bucket -> ((StringStore) bucket.getStore(column)).streamValues()) - .filter(seenValues::add); - - // Chunk values, to produce smaller messages - final Iterable> partition = Iterables.partition(values::iterator, columValueChunkSize); - - log.trace("BEGIN Sending column values for {} in {} sized batches", - column.getId(), columValueChunkSize - ); - - int i = 0; - for (List chunk : partition) { - // Send values to manager - final RegisterColumnValues message = - new RegisterColumnValues(getMessageId(), context.getInfo().getId(), column.getId(), chunk); - - context.send(message); - - log.trace("Finished sending chunk {} for column '{}'", i++, column.getId()); - } - - getProgressReporter().report(1); - log.trace("FINISH collections values for column {} as number {}", column.getId(), done.incrementAndGet()); - }) - ) - .toList(); + final List> futures = columns.stream() + .filter( + column -> table2Buckets.get(column.getTable()) != null) + .map(ColumnId::resolve) + .map( + column -> jobsExecutorService.submit(() -> { + final List buckets = table2Buckets.get(column.getTable().getId()); + + // Keep track of the values we have already seen + HashSet seenValues = new HashSet<>(); + + final Stream values = buckets.stream() + .map(BucketId::resolve) + .flatMap( + bucket -> ((StringStore) bucket.getStore(column)).streamValues()) + .filter(seenValues::add); + + // Chunk values, to produce smaller messages + final Iterable> partition = Iterables.partition( + values::iterator, + columValueChunkSize); + + log.trace( + "BEGIN Sending column values for {} in {} sized batches", + column.getId(), + columValueChunkSize + ); + + int i = 0; + for (List chunk : partition) { + // Send values to manager + final RegisterColumnValues message = new RegisterColumnValues( + getMessageId(), + context.getInfo().getId(), + column.getId(), + chunk); + + context.send(message); + + log.trace("Finished sending chunk {} for column '{}'", i++, column.getId()); + } + + getProgressReporter().report(1); + log.trace( + "FINISH collections values for column {} as number {}", + column.getId(), + done.incrementAndGet()); + }) + ) + .toList(); // We may do this, because we own this specific ExecutorService. jobsExecutorService.shutdown(); while (!jobsExecutorService.awaitTermination(30, TimeUnit.SECONDS)) { - log.debug("Still waiting for jobs on '{}': {} of {} done", context.getInfo().getName(), done.get(), futures.size()); + log.debug( + "Still waiting for jobs on '{}': {} of {} done", + context.getInfo().getName(), + done.get(), + futures.size()); } log.info("FINISH collecting values from {}", Arrays.toString(columns.toArray())); @@ -150,7 +165,9 @@ public String getLabel() { @Override public void afterAllReaction() { - namespace.getJobManager().addSlowJob(namespace.getFilterSearch().createFinalizeFilterSearchJob(namespace, columns)); + namespace.getJobManager() + .addSlowJob( + namespace.getFilterSearch().createFinalizeFilterSearchJob(namespace, columns)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteForm.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteForm.java index 692041f8c5b..65e6eb9d451 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteForm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteForm.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.models.messages.namespaces.specific; -import static com.bakdata.conquery.models.error.ConqueryError.asConqueryError; import java.util.Map; import java.util.Map.Entry; @@ -8,7 +7,6 @@ import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.messages.namespaces.NamespacedMessage; import com.bakdata.conquery.models.messages.namespaces.WorkerMessage; @@ -58,17 +56,20 @@ public void react(Worker worker) throws Exception { // Before we start the query, we create it once to test if it will succeed before creating it multiple times for evaluation per core. try { - query.createQueryPlan(new QueryPlanContext(worker.getStorage(), queryExecutor.getSecondaryIdSubPlanLimit())); - } - catch (Exception e) { + query.createQueryPlan( + new QueryPlanContext(worker.getStorage(), queryExecutor.getSecondaryIdSubPlanLimit())); + } catch (Exception e) { log.warn("Failed to create query plans for {}.", formId, e); queryExecutor.sendFailureToManagerNode(e, formId); return; } - final QueryExecutionContext - subQueryContext = - new QueryExecutionContext(formId, queryExecutor, worker.getStorage(), worker.getBucketManager(), worker.getClock()); + final QueryExecutionContext subQueryContext = new QueryExecutionContext( + formId, + queryExecutor, + worker.getStorage(), + worker.getBucketManager(), + worker.getClock()); Set entities = query.collectRequiredEntities(subQueryContext).resolve(worker.getBucketManager()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteQuery.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteQuery.java index 3336e47390d..bdc367c9866 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteQuery.java @@ -1,12 +1,10 @@ package com.bakdata.conquery.models.messages.namespaces.specific; -import static com.bakdata.conquery.models.error.ConqueryError.asConqueryError; import java.util.Set; import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.messages.namespaces.NamespacedMessage; import com.bakdata.conquery.models.messages.namespaces.WorkerMessage; @@ -52,16 +50,21 @@ public void react(Worker worker) throws Exception { // Before we start the query, we create it once to test if it will succeed before creating it multiple times for evaluation per core. try { Stopwatch stopwatch = Stopwatch.createStarted(); - query.createQueryPlan(new QueryPlanContext(worker.getStorage(), queryExecutor.getSecondaryIdSubPlanLimit())); + query.createQueryPlan( + new QueryPlanContext(worker.getStorage(), queryExecutor.getSecondaryIdSubPlanLimit())); log.trace("Created query plan in {}", stopwatch); - } - catch (Exception e) { + } catch (Exception e) { log.warn("Failed to create query plans for {}.", id, e); queryExecutor.sendFailureToManagerNode(e, id); return; } - final QueryExecutionContext executionContext = new QueryExecutionContext(id, queryExecutor, worker.getStorage(), worker.getBucketManager(), worker.getClock()); + final QueryExecutionContext executionContext = new QueryExecutionContext( + id, + queryExecutor, + worker.getStorage(), + worker.getBucketManager(), + worker.getClock()); final Set entities = query.collectRequiredEntities(executionContext).resolve(worker.getBucketManager()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ImportBucket.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ImportBucket.java index 783ff6a7f2a..02d61012b09 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ImportBucket.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ImportBucket.java @@ -31,14 +31,15 @@ public void react(Worker context) throws Exception { } context.getJobManager() - .addSlowJob(new SimpleJob( - "Adding Bucket %s".formatted(bucket.getId()), - () -> { - log.debug("BEGIN adding Bucket {}", bucket.getId()); - context.addBucket(bucket); - log.trace("DONE adding Bucket {}", bucket.getId()); - } - )); + .addSlowJob( + new SimpleJob( + "Adding Bucket %s".formatted(bucket.getId()), + () -> { + log.debug("BEGIN adding Bucket {}", bucket.getId()); + context.addBucket(bucket); + log.trace("DONE adding Bucket {}", bucket.getId()); + } + )); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/RegisterColumnValues.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/RegisterColumnValues.java index 8daa0c09593..364aaca7184 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/RegisterColumnValues.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/RegisterColumnValues.java @@ -1,5 +1,8 @@ package com.bakdata.conquery.models.messages.namespaces.specific; +import java.util.Collection; +import java.util.UUID; + import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId; import com.bakdata.conquery.models.identifiable.ids.specific.WorkerId; @@ -14,9 +17,6 @@ import lombok.ToString; import lombok.extern.slf4j.Slf4j; -import java.util.Collection; -import java.util.UUID; - /** * This message returns the result of the {@link CollectColumnValuesMessage} to the namespace on the manager. */ diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/RemoveConcept.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/RemoveConcept.java index d061dcbde80..b69fc2f0dbb 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/RemoveConcept.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/RemoveConcept.java @@ -10,10 +10,12 @@ import lombok.RequiredArgsConstructor; import lombok.ToString; -@CPSType(id="REMOVE_CONCEPT", base=NamespacedMessage.class) -@RequiredArgsConstructor(onConstructor_=@JsonCreator) @Getter @ToString +@CPSType(id = "REMOVE_CONCEPT", base = NamespacedMessage.class) +@RequiredArgsConstructor(onConstructor_ = @JsonCreator) +@Getter +@ToString public class RemoveConcept extends WorkerMessage { - + private final ConceptId concept; @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/RemoveImportJob.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/RemoveImportJob.java index 6fa6bd86b59..37a4beca265 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/RemoveImportJob.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/RemoveImportJob.java @@ -11,8 +11,8 @@ import lombok.ToString; import lombok.extern.slf4j.Slf4j; -@CPSType(id="REMOVE_IMPORT", base= NamespacedMessage.class) -@RequiredArgsConstructor(onConstructor_=@JsonCreator) +@CPSType(id = "REMOVE_IMPORT", base = NamespacedMessage.class) +@RequiredArgsConstructor(onConstructor_ = @JsonCreator) @ToString @Slf4j @Getter // Needed by SmileParser diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/RemoveSecondaryId.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/RemoveSecondaryId.java index 1200eeb3e6f..a67b08a1e2b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/RemoveSecondaryId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/RemoveSecondaryId.java @@ -12,8 +12,12 @@ import lombok.ToString; import lombok.extern.slf4j.Slf4j; -@CPSType(id="REMOVE_SECONDARYID", base=NamespacedMessage.class) @Slf4j -@AllArgsConstructor(onConstructor_=@JsonCreator) @Getter @Setter @ToString(callSuper=true) +@CPSType(id = "REMOVE_SECONDARYID", base = NamespacedMessage.class) +@Slf4j +@AllArgsConstructor(onConstructor_ = @JsonCreator) +@Getter +@Setter +@ToString(callSuper = true) public class RemoveSecondaryId extends WorkerMessage { private SecondaryIdDescriptionId secondaryId; diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/RemoveTable.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/RemoveTable.java index 6faade2ac5e..3f53f71ba09 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/RemoveTable.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/RemoveTable.java @@ -12,8 +12,12 @@ import lombok.ToString; import lombok.extern.slf4j.Slf4j; -@CPSType(id="REMOVE_TABLE", base=NamespacedMessage.class) @Slf4j -@AllArgsConstructor(onConstructor_=@JsonCreator) @Getter @Setter @ToString(callSuper=true) +@CPSType(id = "REMOVE_TABLE", base = NamespacedMessage.class) +@Slf4j +@AllArgsConstructor(onConstructor_ = @JsonCreator) +@Getter +@Setter +@ToString(callSuper = true) public class RemoveTable extends WorkerMessage { private TableId table; diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ReportConsistency.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ReportConsistency.java index b7987596c17..064e0b7e63b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ReportConsistency.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ReportConsistency.java @@ -58,22 +58,41 @@ public void react(DistributedNamespace context) throws Exception { throw new IllegalStateException("Detected inconsistency between manager and worker [" + workerId + "]"); } - private static > boolean isConsistent(String typeName, @NonNull Set managerIds, @NonNull Set workerIds, WorkerId workerId) { + private static > boolean isConsistent( + String typeName, + @NonNull Set managerIds, + @NonNull Set workerIds, + WorkerId workerId) { Sets.SetView notInWorker = Sets.difference(managerIds, workerIds); Sets.SetView notInManager = Sets.difference(workerIds, managerIds); if (notInWorker.isEmpty() && notInManager.isEmpty()) { - log.info("{} of worker {} are consistent with the manager: {} {}", typeName, workerId, managerIds.size(), typeName); + log.info( + "{} of worker {} are consistent with the manager: {} {}", + typeName, + workerId, + managerIds.size(), + typeName); return true; } StringBuilder sb = new StringBuilder(); sb.append("Found inconsistencies for ").append(typeName).append(":\n"); for (ID difference : notInWorker) { - sb.append("\t[").append(difference).append("] is not present on the worker but on the manager [").append(workerId).append("].\n"); + sb.append("\t[") + .append(difference) + .append("] is not present on the worker but on the manager [") + .append( + workerId) + .append("].\n"); } for (ID difference : notInManager) { - sb.append("\t[").append(difference).append("] is not present on the manager but on the worker [").append(workerId).append("].\n"); + sb.append("\t[") + .append(difference) + .append("] is not present on the manager but on the worker [") + .append( + workerId) + .append("].\n"); } log.error(sb.toString()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateConcept.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateConcept.java index 8ad428d28e9..bf57860dd9a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateConcept.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateConcept.java @@ -1,5 +1,8 @@ package com.bakdata.conquery.models.messages.namespaces.specific; +import jakarta.validation.Validator; +import jakarta.validation.constraints.NotNull; + import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.exceptions.ValidatorHelper; @@ -10,8 +13,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.OptBoolean; -import jakarta.validation.Validator; -import jakarta.validation.constraints.NotNull; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -19,8 +20,11 @@ import lombok.ToString; import lombok.extern.slf4j.Slf4j; -@CPSType(id="UPDATE_CONCEPT", base=NamespacedMessage.class) -@RequiredArgsConstructor(onConstructor_=@JsonCreator) @Getter @Setter @ToString +@CPSType(id = "UPDATE_CONCEPT", base = NamespacedMessage.class) +@RequiredArgsConstructor(onConstructor_ = @JsonCreator) +@Getter +@Setter +@ToString @Slf4j public class UpdateConcept extends WorkerMessage { @@ -31,7 +35,7 @@ public class UpdateConcept extends WorkerMessage { @JsonIgnore @NotNull private Validator validator; - + @Override public void react(Worker context) throws Exception { ValidatorHelper.failOnError(log, validator.validate(concept)); diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateElementMatchingStats.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateElementMatchingStats.java index ffd7afb6759..7b3fdf4e4c1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateElementMatchingStats.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateElementMatchingStats.java @@ -45,7 +45,9 @@ public void react(DistributedNamespace context) throws Exception { // mapping function cannot use Id::resolve here yet, somehow the nsIdResolver is not set because it // stems from a map key. Jackson seems to use a different serializer. - Concept concept = conceptsToUpdate.computeIfAbsent(conceptId, id -> context.getStorage().getConcept(id)); + Concept concept = conceptsToUpdate.computeIfAbsent( + conceptId, + id -> context.getStorage().getConcept(id)); final ConceptElement target = concept.findById(element); @@ -60,7 +62,10 @@ public void react(DistributedNamespace context) throws Exception { } matchingStats.putEntry(sourceString, value); } catch (Exception e) { - log.error("Failed to set matching stats for '{}' (enable TRACE for exception)", entry.getKey(), (Exception) (log.isTraceEnabled() ? e : null)); + log.error( + "Failed to set matching stats for '{}' (enable TRACE for exception)", + entry.getKey(), + (Exception) (log.isTraceEnabled() ? e : null)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateMatchingStatsMessage.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateMatchingStatsMessage.java index 273bf36cfa0..8d988bd8b66 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateMatchingStatsMessage.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateMatchingStatsMessage.java @@ -23,7 +23,6 @@ import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.events.CBlock; -import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.identifiable.ids.specific.CBlockId; import com.bakdata.conquery.models.identifiable.ids.specific.ConceptElementId; import com.bakdata.conquery.models.identifiable.ids.specific.ConceptId; @@ -79,14 +78,11 @@ private static CDateRange spannedValidityDates(Bucket bucket, int event, Iterabl if (minDate == Integer.MAX_VALUE && maxDate == Integer.MIN_VALUE) { span = null; - } - else if (minDate == Integer.MAX_VALUE) { + } else if (minDate == Integer.MAX_VALUE) { span = CDateRange.atMost(maxDate); - } - else if (maxDate == Integer.MIN_VALUE) { + } else if (maxDate == Integer.MIN_VALUE) { span = CDateRange.atLeast(minDate); - } - else { + } else { span = CDateRange.of(minDate, maxDate); } return span; @@ -105,7 +101,7 @@ private static class UpdateMatchingStatsJob extends Job { @Override public void execute() throws Exception { - try(Stream allCBlocks = worker.getStorage().getAllCBlocks()) { + try (Stream allCBlocks = worker.getStorage().getAllCBlocks()) { if (allCBlocks.findAny().isEmpty()) { log.debug("Worker {} is empty, skipping.", worker); return; @@ -117,34 +113,41 @@ public void execute() throws Exception { log.info("BEGIN update Matching stats for {} Concepts", concepts.size()); - final Map> - subJobs = - concepts.stream() - .collect(Collectors.toMap(Functions.identity(), - conceptId -> CompletableFuture.runAsync(() -> { + final Map> subJobs = concepts.stream() + .collect( + Collectors.toMap( + Functions.identity(), + conceptId -> CompletableFuture.runAsync(() -> { - Map, MatchingStats.Entry> matchingStats = calculateConceptMatches(conceptId, worker); + Map, MatchingStats.Entry> matchingStats = calculateConceptMatches( + conceptId, + worker); - final WriteFuture writeFuture = worker.send(new UpdateElementMatchingStats(worker.getInfo().getId(), matchingStats)); + final WriteFuture writeFuture = worker.send( + new UpdateElementMatchingStats(worker.getInfo().getId(), matchingStats)); - progressReporter.report(1); - }, worker.getJobsExecutorService()) - )); + progressReporter.report(1); + }, worker.getJobsExecutorService()) + )); log.debug("All jobs submitted. Waiting for completion."); - final CompletableFuture all = CompletableFuture.allOf(subJobs.values().toArray(CompletableFuture[]::new)); + final CompletableFuture all = CompletableFuture.allOf( + subJobs.values().toArray(CompletableFuture[]::new)); do { try { all.get(1, TimeUnit.MINUTES); - } - catch (TimeoutException exception) { + } catch (TimeoutException exception) { // Count unfinished matching stats jobs. if (log.isDebugEnabled()) { - final long unfinished = subJobs.values().stream().filter(Predicate.not(CompletableFuture::isDone)).count(); + final long unfinished = subJobs.values() + .stream() + .filter( + Predicate.not(CompletableFuture::isDone)) + .count(); log.debug("{} still waiting for {} tasks", worker.getInfo().getDataset(), unfinished); } @@ -171,11 +174,13 @@ public String getLabel() { return String.format("Calculate Matching Stats for %s", worker.getInfo().getDataset()); } - private static Map, MatchingStats.Entry> calculateConceptMatches(ConceptId conceptId, Worker worker) { + private static Map, MatchingStats.Entry> calculateConceptMatches( + ConceptId conceptId, + Worker worker) { Concept concept = conceptId.resolve(); - final Map, MatchingStats.Entry> matchingStats = new HashMap<>(concept.countElements()); + final Map, MatchingStats.Entry> matchingStats = new HashMap<>(concept.countElements()); log.debug("BEGIN calculating for `{}`", conceptId); @@ -191,7 +196,10 @@ private static Map, MatchingStats.Entry> calculateConceptMat CBlock cBlock = cBlockId.resolve(); final Bucket bucket = cBlock.getBucket().resolve(); final Table table = bucket.getTable().resolve(); - final List dateColumns = Arrays.stream(table.getColumns()).filter(t -> t.getType().isDateCompatible()).toList(); + final List dateColumns = Arrays.stream(table.getColumns()) + .filter( + t -> t.getType().isDateCompatible()) + .toList(); for (String entity : bucket.entities()) { @@ -204,8 +212,9 @@ private static Map, MatchingStats.Entry> calculateConceptMat if (!(concept instanceof TreeConcept) || localIds == null) { - matchingStats.computeIfAbsent(conceptId, (ignored) -> new MatchingStats.Entry()) - .addEvents(entity, 1, span); + matchingStats.computeIfAbsent( + conceptId, + (ignored) -> new MatchingStats.Entry()).addEvents(entity, 1, span); continue; } @@ -216,15 +225,15 @@ private static Map, MatchingStats.Entry> calculateConceptMat ConceptElement element = ((TreeConcept) concept).getElementByLocalIdPath(localIds); while (element != null) { - matchingStats.computeIfAbsent(element.getId(), (ignored) -> new MatchingStats.Entry()) - .addEvents(entity, 1, span); + matchingStats.computeIfAbsent( + element.getId(), + (ignored) -> new MatchingStats.Entry()).addEvents(entity, 1, span); element = element.getParent(); } } } - } - catch (Exception e) { + } catch (Exception e) { log.error("Failed to collect the matching stats for {}", cBlockId, e); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateSecondaryId.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateSecondaryId.java index 51f83d0d32a..daba76b1f93 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateSecondaryId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateSecondaryId.java @@ -12,8 +12,12 @@ import lombok.ToString; import lombok.extern.slf4j.Slf4j; -@CPSType(id="UPDATE_SECONDARYID", base=NamespacedMessage.class) @Slf4j -@AllArgsConstructor(onConstructor_=@JsonCreator) @Getter @Setter @ToString(callSuper=true) +@CPSType(id = "UPDATE_SECONDARYID", base = NamespacedMessage.class) +@Slf4j +@AllArgsConstructor(onConstructor_ = @JsonCreator) +@Getter +@Setter +@ToString(callSuper = true) public class UpdateSecondaryId extends WorkerMessage { private SecondaryIdDescription secondaryId; diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateTable.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateTable.java index 224d7b213e4..578cde69721 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateTable.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateTable.java @@ -12,8 +12,12 @@ import lombok.ToString; import lombok.extern.slf4j.Slf4j; -@CPSType(id="UPDATE_TABLE", base=NamespacedMessage.class) @Slf4j -@AllArgsConstructor(onConstructor_=@JsonCreator) @Getter @Setter @ToString(callSuper=true) +@CPSType(id = "UPDATE_TABLE", base = NamespacedMessage.class) +@Slf4j +@AllArgsConstructor(onConstructor_ = @JsonCreator) +@Getter +@Setter +@ToString(callSuper = true) public class UpdateTable extends WorkerMessage { private Table table; diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateWorkerBucket.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateWorkerBucket.java index c985000270e..6be76201daf 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateWorkerBucket.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateWorkerBucket.java @@ -20,7 +20,10 @@ public class UpdateWorkerBucket extends WorkerMessage { @Override public void react(Worker context) throws Exception { - log.debug("Received new WorkerInformation(size = {},dataset = {})", info.getIncludedBuckets().size(), info.getDataset()); + log.debug( + "Received new WorkerInformation(size = {},dataset = {})", + info.getIncludedBuckets().size(), + info.getDataset()); //new included buckets from master context.updateWorkerInfo(info); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/network/MessageToManagerNode.java b/backend/src/main/java/com/bakdata/conquery/models/messages/network/MessageToManagerNode.java index 3214050caf0..64bf5c50023 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/network/MessageToManagerNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/network/MessageToManagerNode.java @@ -9,7 +9,9 @@ public abstract class MessageToManagerNode extends NetworkMessage { public static abstract class Slow extends MessageToManagerNode implements SlowMessage { - @JsonIgnore @Getter @Setter + @JsonIgnore + @Getter + @Setter private ProgressReporter progressReporter; } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/network/MessageToShardNode.java b/backend/src/main/java/com/bakdata/conquery/models/messages/network/MessageToShardNode.java index 35853dd143e..e0ca92ea539 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/network/MessageToShardNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/network/MessageToShardNode.java @@ -9,7 +9,9 @@ public abstract class MessageToShardNode extends NetworkMessage { public static abstract class Slow extends MessageToShardNode implements SlowMessage { - @JsonIgnore @Getter @Setter + @JsonIgnore + @Getter + @Setter private ProgressReporter progressReporter; } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/network/NetworkMessage.java b/backend/src/main/java/com/bakdata/conquery/models/messages/network/NetworkMessage.java index bb18235b261..81d5c3ba5ca 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/network/NetworkMessage.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/network/NetworkMessage.java @@ -1,7 +1,7 @@ package com.bakdata.conquery.models.messages.network; -import java.util.UUID; import jakarta.validation.constraints.NotNull; +import java.util.UUID; import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.models.messages.Message; diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/network/NetworkMessageContext.java b/backend/src/main/java/com/bakdata/conquery/models/messages/network/NetworkMessageContext.java index 9e29075a044..2677eebce80 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/network/NetworkMessageContext.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/network/NetworkMessageContext.java @@ -36,7 +36,11 @@ public static class ShardNodeNetworkContext extends NetworkMessageContext datasetRegistry; - public ManagerNodeNetworkContext(NetworkSession session, DatasetRegistry datasetRegistry, ClusterState clusterState) { + public ManagerNodeNetworkContext( + NetworkSession session, + DatasetRegistry datasetRegistry, + ClusterState clusterState) { super(session); this.datasetRegistry = datasetRegistry; this.clusterState = clusterState; diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/AddWorker.java b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/AddWorker.java index 780f2a28c14..1282a460e4b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/AddWorker.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/AddWorker.java @@ -27,8 +27,13 @@ public void react(ShardNodeNetworkContext context) throws Exception { dataset.setStorageProvider(context.getWorkers()); - Worker worker = - context.getWorkers().newWorker(dataset, createWorkerName(), context.getSession(), context.getConfig().getStorage(), context.getConfig().isFailOnError()); + Worker worker = context.getWorkers() + .newWorker( + dataset, + createWorkerName(), + context.getSession(), + context.getConfig().getStorage(), + context.getConfig().isFailOnError()); context.send(new RegisterWorker(worker.getInfo())); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/CancelJobMessage.java b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/CancelJobMessage.java index d70d59d44a4..431cb01a582 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/CancelJobMessage.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/CancelJobMessage.java @@ -10,7 +10,7 @@ import lombok.Data; import lombok.RequiredArgsConstructor; -@CPSType(id="CANCEL_JOB", base= NetworkMessage.class) +@CPSType(id = "CANCEL_JOB", base = NetworkMessage.class) @Data @RequiredArgsConstructor(onConstructor_ = @JsonCreator) public class CancelJobMessage extends MessageToShardNode { diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/ForwardToNamespace.java b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/ForwardToNamespace.java index e24003d19f1..45702029717 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/ForwardToNamespace.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/ForwardToNamespace.java @@ -29,7 +29,9 @@ public class ForwardToNamespace extends MessageToManagerNode implements SlowMess public void react(ManagerNodeNetworkContext context) throws Exception { - DistributedNamespace ns = Objects.requireNonNull(context.getDatasetRegistry().get(datasetId), () -> String.format("Missing dataset `%s`", datasetId)); + DistributedNamespace ns = Objects.requireNonNull( + context.getDatasetRegistry().get(datasetId), + () -> String.format("Missing dataset `%s`", datasetId)); ConqueryMDC.setLocation(ns.getStorage().getDataset().toString()); message.react(ns); diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/RegisterWorker.java b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/RegisterWorker.java index 985e410c3b1..de7fe86953d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/RegisterWorker.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/RegisterWorker.java @@ -11,19 +11,20 @@ import lombok.Data; import lombok.RequiredArgsConstructor; -@CPSType(id="REGISTER_SHARD_WORKER_IDENTITY", base=NetworkMessage.class) +@CPSType(id = "REGISTER_SHARD_WORKER_IDENTITY", base = NetworkMessage.class) @Data @RequiredArgsConstructor(onConstructor_ = @JsonCreator) public class RegisterWorker extends MessageToManagerNode { private final WorkerInformation info; - + @Override public void react(ManagerNodeNetworkContext context) throws Exception { ShardNodeInformation node = getShardNode(context); - - if(node == null) { - throw new IllegalStateException("Received worker %s from unknown shard %s".formatted(info.getId(), context.getRemoteAddress())); + + if (node == null) { + throw new IllegalStateException( + "Received worker %s from unknown shard %s".formatted(info.getId(), context.getRemoteAddress())); } info.setConnectedShardNode(node); @@ -39,8 +40,6 @@ public void react(ManagerNodeNetworkContext context) throws Exception { * @return the found slave or null if none was found */ private ShardNodeInformation getShardNode(ManagerNodeNetworkContext context) { - return context.getClusterState() - .getShardNodes() - .get(context.getRemoteAddress()); + return context.getClusterState().getShardNodes().get(context.getRemoteAddress()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/RemoveWorker.java b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/RemoveWorker.java index 31cdfa16b7f..5b8f1aa357f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/RemoveWorker.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/RemoveWorker.java @@ -10,14 +10,14 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -@CPSType(id="REMOVE_WORKER", base=NetworkMessage.class) +@CPSType(id = "REMOVE_WORKER", base = NetworkMessage.class) @RequiredArgsConstructor(onConstructor_ = @JsonCreator) @Data @Slf4j public class RemoveWorker extends MessageToShardNode.Slow { private final DatasetId dataset; - + @Override public void react(ShardNodeNetworkContext context) throws Exception { log.info("Removing worker {}", dataset); diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/UpdateJobManagerStatus.java b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/UpdateJobManagerStatus.java index 504e5de2c05..2869353b5d5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/UpdateJobManagerStatus.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/UpdateJobManagerStatus.java @@ -26,13 +26,14 @@ public void react(ManagerNodeNetworkContext context) throws Exception { final ShardNodeInformation node = context.getClusterState().getShardNodes().get(context.getRemoteAddress()); if (node == null) { - log.error("Could not find ShardNode `{}`, I only know of {}", context.getRemoteAddress(), context.getClusterState().getShardNodes().keySet()); + log.error( + "Could not find ShardNode `{}`, I only know of {}", + context.getRemoteAddress(), + context.getClusterState().getShardNodes().keySet()); return; } // The shards don't know their own name so we attach it here - node.addJobManagerStatus(status.toBuilder() - .origin(context.getRemoteAddress().toString()) - .build()); + node.addJobManagerStatus(status.toBuilder().origin(context.getRemoteAddress().toString()).build()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/GroovyPredicate.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/GroovyPredicate.java index 69083867f44..5741e4b4504 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/GroovyPredicate.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/GroovyPredicate.java @@ -9,7 +9,7 @@ @Slf4j public abstract class GroovyPredicate extends Script { - public boolean filterRow(String[] row){ + public boolean filterRow(String[] row) { setProperty("row", row); log.trace("Row = {}, length = {}", row, row.length); return run(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/PPColumn.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/PPColumn.java index f3e00d8be3e..74e8c941bf5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/PPColumn.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/PPColumn.java @@ -12,7 +12,7 @@ @Slf4j @Data -@RequiredArgsConstructor(onConstructor_ = {@JsonCreator} ) +@RequiredArgsConstructor(onConstructor_ = {@JsonCreator}) public class PPColumn { @NonNull private final String name; diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/Preprocessed.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/Preprocessed.java index e32f51b359e..5df925853e1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/Preprocessed.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/Preprocessed.java @@ -101,16 +101,26 @@ public void write(File file, int buckets) throws IOException { log.debug("Writing Headers"); //TODO this could actually be done at read-time, avoiding large allocations entirely. But in a different smaller PR. - final Map> bucket2Entity = entityStart.keySet().stream() - .collect(Collectors.groupingBy(id -> getEntityBucket(buckets, id))) - .entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> new HashSet<>(entry.getValue()))); + final Map> bucket2Entity = entityStart.keySet() + .stream() + .collect( + Collectors.groupingBy(id -> getEntityBucket(buckets, id))) + .entrySet() + .stream() + .collect( + Collectors.toMap(Map.Entry::getKey, entry -> new HashSet<>(entry.getValue()))); final int hash = descriptor.calculateValidityHash(job.getCsvDirectory(), job.getTag()); - final PreprocessedHeader header = - new PreprocessedHeader(descriptor.getName(), descriptor.getTable(), rows, entityStart.size(), bucket2Entity.size(), columns, hash); + final PreprocessedHeader header = new PreprocessedHeader( + descriptor.getName(), + descriptor.getTable(), + rows, + entityStart.size(), + bucket2Entity.size(), + columns, + hash); writePreprocessed(file, header, entityStart, entityLength, columnStores, bucket2Entity); } @@ -144,7 +154,10 @@ private void calculateEntitySpans(Object2IntMap entityStart, Object2IntM */ @SuppressWarnings("rawtypes") private Map combineStores(Object2IntMap entityStart) { - final Map columnStores = Arrays.stream(columns).parallel().collect(Collectors.toMap(PPColumn::getName, PPColumn::findBestType)); + final Map columnStores = Arrays.stream(columns) + .parallel() + .collect( + Collectors.toMap(PPColumn::getName, PPColumn::findBestType)); // This object can be huge! final Map entityEvents = new HashMap<>(entityStart.size()); @@ -173,8 +186,7 @@ private Map combineStores(Object2IntMap entityStart for (int inIndex : events) { if (columnValues.isNull(inIndex)) { store.setNull(outIndex); - } - else { + } else { final Object raw = columnValues.get(inIndex); ppColumn.getParser().setValue(store, outIndex, raw); } @@ -185,9 +197,19 @@ private Map combineStores(Object2IntMap entityStart return columnStores; } - private static void writePreprocessed(File file, PreprocessedHeader header, Map globalStarts, Map globalLengths, Map data, Map> bucket2Entities) throws IOException { + private static void writePreprocessed( + File file, + PreprocessedHeader header, + Map globalStarts, + Map globalLengths, + Map data, + Map> bucket2Entities) throws IOException { final OutputStream out = new GZIPOutputStream(new FileOutputStream(file)); - try (JsonGenerator generator = Jackson.BINARY_MAPPER.copy().enable(JsonGenerator.Feature.AUTO_CLOSE_TARGET).getFactory().createGenerator(out)) { + try (JsonGenerator generator = Jackson.BINARY_MAPPER.copy() + .enable( + JsonGenerator.Feature.AUTO_CLOSE_TARGET) + .getFactory() + .createGenerator(out)) { log.debug("Writing header"); @@ -208,7 +230,11 @@ private static void writePreprocessed(File file, PreprocessedHeader header, Map< } } - private static PreprocessedData selectBucket(int bucket, Map localStarts, Map localLengths, Map stores) { + private static PreprocessedData selectBucket( + int bucket, + Map localStarts, + Map localLengths, + Map stores) { final IntList selectionStart = new IntArrayList(); @@ -261,7 +287,10 @@ public synchronized void addRow(String primaryId, PPColumn[] columns, Object[] o for (int col = 0; col < outRow.length; col++) { if (values[col] == null && outRow[col] != null) { - throw new IllegalStateException(String.format("Expecting %s to be NULL, because no ColumnValues could be generated by the associated parser", outRow[col])); + throw new IllegalStateException( + String.format( + "Expecting %s to be NULL, because no ColumnValues could be generated by the associated parser", + outRow[col])); } if (values[col] == null) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/PreprocessedHeader.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/PreprocessedHeader.java index 83d2674f960..cc278738f6c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/PreprocessedHeader.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/PreprocessedHeader.java @@ -25,7 +25,8 @@ * @implSpec The Columns and their order must directly match the layout in the data. */ @Data -@Getter @Setter +@Getter +@Setter @RequiredArgsConstructor @Slf4j public class PreprocessedHeader { @@ -69,7 +70,11 @@ public Import createImportDescription(TableId tableId, Map for (int i = 0; i < columns.length; i++) { final ColumnStore store = stores.get(columns[i].getName()); - final ImportColumn col = new ImportColumn(imp, store.createDescription(), store.getLines(), numberOfBuckets * store.estimateMemoryConsumptionBytes()); + final ImportColumn col = new ImportColumn( + imp, + store.createDescription(), + store.getLines(), + numberOfBuckets * store.estimateMemoryConsumptionBytes()); col.setName(columns[i].getName()); @@ -89,10 +94,15 @@ public List assertMatch(Table table) { final List errors = new ArrayList<>(); if (table.getColumns().length != getColumns().length) { - errors.add("Import column count=`%d` does not match table column count=`%d`".formatted(getColumns().length, table.getColumns().length)); + errors.add( + "Import column count=`%d` does not match table column count=`%d`".formatted( + getColumns().length, + table.getColumns().length)); } - final Map typesByName = Arrays.stream(getColumns()).collect(Collectors.toMap(PPColumn::getName, PPColumn::getType)); + final Map typesByName = Arrays.stream(getColumns()) + .collect( + Collectors.toMap(PPColumn::getName, PPColumn::getType)); for (int i = 0; i < Math.min(table.getColumns().length, getColumns().length); i++) { final Column column = table.getColumns()[i]; @@ -103,7 +113,11 @@ public List assertMatch(Table table) { } if (!typesByName.get(column.getName()).equals(column.getType())) { - errors.add("Column[%s] Types do not match %s != %s".formatted(column.getName(), typesByName.get(column.getName()), column.getType())); + errors.add( + "Column[%s] Types do not match %s != %s".formatted( + column.getName(), + typesByName.get(column.getName()), + column.getType())); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/PreprocessedReader.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/PreprocessedReader.java index 5f180edf280..2442b978406 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/PreprocessedReader.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/PreprocessedReader.java @@ -24,12 +24,10 @@ public class PreprocessedReader implements AutoCloseable, Iterator * Reads CSV file, per row extracts the primary key, then applies other transformations on each row, then compresses the data with {@link ColumnStore}. */ - public static void preprocess(PreprocessingJob preprocessingJob, ProgressBar totalProgress, ConqueryConfig config, int buckets) throws IOException { + public static void preprocess( + PreprocessingJob preprocessingJob, + ProgressBar totalProgress, + ConqueryConfig config, + int buckets) throws IOException { final File preprocessedFile = preprocessingJob.getPreprocessedFile(); TableImportDescriptor descriptor = preprocessingJob.getDescriptor(); @@ -70,9 +76,8 @@ public static void preprocess(PreprocessingJob preprocessingJob, ProgressBar tot } if (!Files.isWritable(preprocessedFile.toPath().getParent())) { - throw new IllegalArgumentException("No write permission in " + LogUtil.printPath(preprocessedFile - .toPath() - .getParent())); + throw new IllegalArgumentException( + "No write permission in " + LogUtil.printPath(preprocessedFile.toPath().getParent())); } //delete target file if it exists @@ -95,9 +100,17 @@ public static void preprocess(PreprocessingJob preprocessingJob, ProgressBar tot for (int inputSource = 0; inputSource < descriptor.getInputs().length; inputSource++) { final TableInputDescriptor input = descriptor.getInputs()[inputSource]; - final File sourceFile = resolveSourceFile(input.getSourceFile(), preprocessingJob.getCsvDirectory(), preprocessingJob.getTag()); - - final String name = String.format("%s:%s[%d/%s]", descriptor.toString(), descriptor.getTable(), inputSource, sourceFile.getName()); + final File sourceFile = resolveSourceFile( + input.getSourceFile(), + preprocessingJob.getCsvDirectory(), + preprocessingJob.getTag()); + + final String name = String.format( + "%s:%s[%d/%s]", + descriptor.toString(), + descriptor.getTable(), + inputSource, + sourceFile.getName()); ConqueryMDC.setLocation(name); if (!(sourceFile.exists() && sourceFile.canRead())) { @@ -114,7 +127,9 @@ public static void preprocess(PreprocessingJob preprocessingJob, ProgressBar tot // Create CSV parser according to config, but overriding some behaviour. parser = csvSettings.withParseHeaders(true).withSkipHeader(false).createParser(); - parser.beginParsing(FileUtil.isGZipped(sourceFile) ? new GZIPInputStream(countingIn) : countingIn, csvSettings.getEncoding()); + parser.beginParsing( + FileUtil.isGZipped(sourceFile) ? new GZIPInputStream(countingIn) : countingIn, + csvSettings.getEncoding()); final String[] headers = parser.getContext().parsedHeaders(); @@ -125,7 +140,11 @@ public static void preprocess(PreprocessingJob preprocessingJob, ProgressBar tot DateReader dateReader = config.getLocale().getDateReader(); - final OutputDescription.Output primaryOut = input.getPrimary().createForHeaders(headerMap, dateReader, config); + final OutputDescription.Output primaryOut = input.getPrimary() + .createForHeaders( + headerMap, + dateReader, + config); final List outputs = new ArrayList<>(); final PPColumn[] columns = result.getColumns(); @@ -146,8 +165,9 @@ public static void preprocess(PreprocessingJob preprocessingJob, ProgressBar tot } try { - String primaryId = - (String) Objects.requireNonNull(primaryOut.createOutput(row, result.getPrimaryColumn(), lineId), "primaryId may not be null"); + String primaryId = (String) Objects.requireNonNull( + primaryOut.createOutput(row, result.getPrimaryColumn(), lineId), + "primaryId may not be null"); final String primary = result.addPrimary(primaryId); @@ -155,37 +175,37 @@ public static void preprocess(PreprocessingJob preprocessingJob, ProgressBar tot result.addRow(primary, columns, outRow); - } - catch (OutputDescription.OutputException e) { + } catch (OutputDescription.OutputException e) { exceptions.put(e.getCause().getClass(), exceptions.getInt(e.getCause().getClass()) + 1); errors++; if (log.isTraceEnabled() || errors < config.getPreprocessor().getMaximumPrintedErrors()) { - log.warn("Failed to parse `{}` from line: {} content: {}", e.getSource(), lineId, row, e.getCause()); - } - else if (errors == config.getPreprocessor().getMaximumPrintedErrors()) { - log.warn("More erroneous lines occurred. Only the first " - + config.getPreprocessor().getMaximumPrintedErrors() - + " were printed."); + log.warn( + "Failed to parse `{}` from line: {} content: {}", + e.getSource(), + lineId, + row, + e.getCause()); + } else if (errors == config.getPreprocessor().getMaximumPrintedErrors()) { + log.warn( + "More erroneous lines occurred. Only the first " + config.getPreprocessor() + .getMaximumPrintedErrors() + " were printed."); } - } - catch (Exception e) { + } catch (Exception e) { exceptions.put(e.getClass(), exceptions.getInt(e.getClass()) + 1); errors++; if (log.isTraceEnabled() || errors < config.getPreprocessor().getMaximumPrintedErrors()) { log.warn("Failed to parse line: {} content: {}", lineId, row, e); + } else if (errors == config.getPreprocessor().getMaximumPrintedErrors()) { + log.warn( + "More erroneous lines occurred. Only the first " + config.getPreprocessor() + .getMaximumPrintedErrors() + " were printed."); } - else if (errors == config.getPreprocessor().getMaximumPrintedErrors()) { - log.warn("More erroneous lines occurred. Only the first " - + config.getPreprocessor().getMaximumPrintedErrors() - + " were printed."); - } - } - finally { + } finally { //report progress totalProgress.addCurrentValue(countingIn.getCount() - progress); progress = countingIn.getCount(); @@ -193,8 +213,7 @@ else if (errors == config.getPreprocessor().getMaximumPrintedErrors()) { } } - } - finally { + } finally { if (parser != null) { parser.stopParsing(); } @@ -212,7 +231,11 @@ else if (errors == config.getPreprocessor().getMaximumPrintedErrors()) { result.write(tmp, buckets); if (errors > 0) { - log.warn("Had {}% faulty lines ({} of ~{} lines)", String.format("%.2f", 100d * (double) errors / (double) lineId), errors, lineId); + log.warn( + "Had {}% faulty lines ({} of ~{} lines)", + String.format("%.2f", 100d * (double) errors / (double) lineId), + errors, + lineId); } if ((double) errors / (double) lineId > config.getPreprocessor().getFaultyLineThreshold()) { @@ -228,8 +251,11 @@ else if (errors == config.getPreprocessor().getMaximumPrintedErrors()) { /** * Apply each output for a single row. Returning all resulting values. */ - private static Object[] applyOutputs(List outputs, PPColumn[] columns, String[] row, long lineId) - throws ParsingException, OutputDescription.OutputException { + private static Object[] applyOutputs( + List outputs, + PPColumn[] columns, + String[] row, + long lineId) throws ParsingException, OutputDescription.OutputException { Object[] outRow = new Object[outputs.size()]; for (int index = 0; index < outputs.size(); index++) { @@ -245,8 +271,7 @@ private static Object[] applyOutputs(List outputs, PPC } outRow[index] = result; - } - catch (Exception e) { + } catch (Exception e) { throw new OutputDescription.OutputException(out.getDescription(), e); } } @@ -267,15 +292,13 @@ public static File resolveSourceFile(String fileName, Path csvDirectory, Optiona if (name.endsWith(".csv.gz")) { name = name.substring(0, name.length() - ".csv.gz".length()); suffix = ".csv.gz"; - } - else if (name.endsWith(".csv")) { + } else if (name.endsWith(".csv")) { name = name.substring(0, name.length() - ".csv".length()); suffix = ".csv"; - } - else { + } else { throw new IllegalArgumentException("Unknown suffix for file " + name); } return csvDirectory.resolve(name + "." + tag.get() + suffix).toFile(); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/TableImportDescriptor.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/TableImportDescriptor.java index afccff0fb8a..5b1ef5e23f4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/TableImportDescriptor.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/TableImportDescriptor.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.preproc; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; import java.io.File; import java.io.IOException; import java.io.Serializable; @@ -8,8 +10,6 @@ import java.util.Arrays; import java.util.List; import java.util.Optional; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotEmpty; import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.models.events.MajorTypeId; @@ -62,7 +62,8 @@ public static TableImportDescriptor read(File descriptionFile) throws IOExceptio } @JsonIgnore - @ValidationMethod(message = "The output of each input needs the same number of output columns of the same type and name") + @ValidationMethod( + message = "The output of each input needs the same number of output columns of the same type and name") public boolean isSameTypesInEachInput() { if (inputs.length == 1) { return true; @@ -70,7 +71,10 @@ public boolean isSameTypesInEachInput() { List types = new ArrayList<>(); for (TableInputDescriptor input : inputs) { - MajorTypeId[] inp = Arrays.stream(input.getOutput()).map(OutputDescription::getResultType).toArray(MajorTypeId[]::new); + MajorTypeId[] inp = Arrays.stream(input.getOutput()) + .map(OutputDescription::getResultType) + .toArray( + MajorTypeId[]::new); for (MajorTypeId[] out : types) { if (!Arrays.equals(inp, out)) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/TableInputDescriptor.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/TableInputDescriptor.java index 90ad85ee948..cc3825763d7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/TableInputDescriptor.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/TableInputDescriptor.java @@ -1,5 +1,8 @@ package com.bakdata.conquery.models.preproc; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; import java.time.LocalDate; import java.util.Arrays; import java.util.stream.Stream; @@ -14,9 +17,6 @@ import io.dropwizard.validation.ValidationMethod; import it.unimi.dsi.fastutil.objects.Object2IntArrayMap; import it.unimi.dsi.fastutil.objects.Object2IntMap; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.builder.HashCodeBuilder; @@ -36,8 +36,8 @@ public class TableInputDescriptor { private static final long serialVersionUID = 1L; private static final String[] AUTO_IMPORTS = Stream.of( - LocalDate.class, - Range.class + LocalDate.class, + Range.class ).map(Class::getName).toArray(String[]::new); @NotNull @@ -66,9 +66,7 @@ public class TableInputDescriptor { @JsonIgnore @ValidationMethod(message = "One or more columns are duplicated") public boolean allColumnsDistinct() { - return Arrays.stream(getOutput()).map(OutputDescription::getName) - .distinct() - .count() == getOutput().length; + return Arrays.stream(getOutput()).map(OutputDescription::getName).distinct().count() == getOutput().length; } @@ -77,8 +75,7 @@ public boolean allColumnsDistinct() { public boolean isValidGroovyScript() { try { createFilter(FAKE_HEADERS); - } - catch (Exception ex) { + } catch (Exception ex) { log.error("Groovy script is not valid", ex); return false; } @@ -127,8 +124,7 @@ public GroovyPredicate createFilter(String[] headers) { } return (GroovyPredicate) groovy.parse(filter); - } - catch (Exception e) { + } catch (Exception e) { throw new RuntimeException("Failed to compile filter `" + filter + "`", e); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/CompoundDateRangeOutput.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/CompoundDateRangeOutput.java index ec8447a1829..af181061a6c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/CompoundDateRangeOutput.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/CompoundDateRangeOutput.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.preproc.outputs; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; import java.util.Arrays; import com.bakdata.conquery.io.cps.CPSType; @@ -13,8 +15,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; import it.unimi.dsi.fastutil.objects.Object2IntArrayMap; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; import lombok.Data; import lombok.ToString; @@ -36,16 +36,24 @@ public class CompoundDateRangeOutput extends OutputDescription { @Override public Output createForHeaders(Object2IntArrayMap headers, DateReader dateReader, ConqueryConfig config) { final Output startReader = Arrays.stream(getParent().getOutput()) - .filter(output -> output.getName().equals(getStartColumn())) - .findFirst() - .orElseThrow() - .createForHeaders(headers, dateReader, config); + .filter( + output -> output.getName().equals(getStartColumn())) + .findFirst() + .orElseThrow() + .createForHeaders( + headers, + dateReader, + config); final Output endReader = Arrays.stream(getParent().getOutput()) - .filter(output -> output.getName().equals(getEndColumn())) - .findFirst() - .orElseThrow() - .createForHeaders(headers, dateReader, config); + .filter( + output -> output.getName().equals(getEndColumn())) + .findFirst() + .orElseThrow() + .createForHeaders( + headers, + dateReader, + config); final DateParser dateParser = new DateParser(config); @@ -68,9 +76,9 @@ protected Object parseLine(String[] row, Parser type, long sourceLine) throws Pa } return - // Since it's not possible that BOTH are null either of them being null already implies an open and therefore valid range. - (start == null || end == null) - // row is included if start <= end + // Since it's not possible that BOTH are null either of them being null already implies an open and therefore valid range. + (start == null || end == null) + // row is included if start <= end || (Integer) start <= (Integer) end; } }; @@ -87,8 +95,10 @@ protected Object parseLine(String[] row, Parser type, long sourceLine) throws Pa @ValidationMethod(message = "End-column not found") public boolean isEndColumnPresent() { return Arrays.stream(getParent().getOutput()) - .filter(output -> output.getName().equals(getEndColumn())) - .anyMatch(output -> output.getResultType().equals(MajorTypeId.DATE)); + .filter( + output -> output.getName().equals(getEndColumn())) + .anyMatch( + output -> output.getResultType().equals(MajorTypeId.DATE)); } /** @@ -98,8 +108,10 @@ public boolean isEndColumnPresent() { @ValidationMethod(message = "Start-column not found") public boolean isStartColumnPresent() { return Arrays.stream(getParent().getOutput()) - .filter(output -> output.getName().equals(getStartColumn())) - .anyMatch(output -> output.getResultType().equals(MajorTypeId.DATE)); + .filter( + output -> output.getName().equals(getStartColumn())) + .anyMatch( + output -> output.getResultType().equals(MajorTypeId.DATE)); } /** diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/CopyOutput.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/CopyOutput.java index 235996be515..04f99e7f787 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/CopyOutput.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/CopyOutput.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.preproc.outputs; +import jakarta.validation.constraints.NotNull; + import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.events.MajorTypeId; @@ -7,7 +9,6 @@ import com.bakdata.conquery.models.preproc.parser.Parser; import com.bakdata.conquery.util.DateReader; import it.unimi.dsi.fastutil.objects.Object2IntArrayMap; -import jakarta.validation.constraints.NotNull; import lombok.Data; import lombok.NoArgsConstructor; import lombok.ToString; @@ -33,11 +34,7 @@ public CopyOutput(String name, String inputColumn, MajorTypeId typeId) { @Override public int hashCode() { - return new HashCodeBuilder() - .append(super.hashCode()) - .append(inputColumn) - .append(inputType.name()) - .toHashCode(); + return new HashCodeBuilder().append(super.hashCode()).append(inputColumn).append(inputType.name()).toHashCode(); } @NotNull @@ -55,7 +52,11 @@ public Output createForHeaders(Object2IntArrayMap headers, DateReader da return new Output() { @Override protected Object parseLine(String[] row, Parser type, long sourceLine) throws ParsingException { - log.trace("Registering `{}` in line {} for Output[{}]", row[column], sourceLine, this.getDescription().getName()); + log.trace( + "Registering `{}` in line {} for Output[{}]", + row[column], + sourceLine, + this.getDescription().getName()); if (row[column] == null) { return null; diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/DateRangeOutput.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/DateRangeOutput.java index 5e13281afc4..08a5fd15b86 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/DateRangeOutput.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/DateRangeOutput.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.preproc.outputs; +import jakarta.validation.constraints.NotNull; import java.time.LocalDate; import com.bakdata.conquery.io.cps.CPSType; @@ -12,7 +13,6 @@ import com.bakdata.conquery.util.DateReader; import com.google.common.base.Strings; import it.unimi.dsi.fastutil.objects.Object2IntArrayMap; -import jakarta.validation.constraints.NotNull; import lombok.Data; import lombok.ToString; import org.apache.commons.lang3.builder.HashCodeBuilder; @@ -34,13 +34,13 @@ public class DateRangeOutput extends OutputDescription { public boolean allowOpen = false; @Override - public int hashCode(){ - return new HashCodeBuilder() - .append(super.hashCode()) - .append(startColumn) - .append(endColumn) - .append(allowOpen) - .toHashCode(); + public int hashCode() { + return new HashCodeBuilder().append(super.hashCode()) + .append(startColumn) + .append(endColumn) + .append( + allowOpen) + .toHashCode(); } @Override @@ -80,4 +80,4 @@ public MajorTypeId getResultType() { return new DateRangeParser(config); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/EpochDateRangeOutput.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/EpochDateRangeOutput.java index 1a4635a87d8..753c328e61c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/EpochDateRangeOutput.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/EpochDateRangeOutput.java @@ -33,13 +33,13 @@ public class EpochDateRangeOutput extends OutputDescription { private boolean allowOpen = false; @Override - public int hashCode(){ - return new HashCodeBuilder() - .append(super.hashCode()) - .append(startColumn) - .append(endColumn) - .append(allowOpen) - .toHashCode(); + public int hashCode() { + return new HashCodeBuilder().append(super.hashCode()) + .append(startColumn) + .append(endColumn) + .append( + allowOpen) + .toHashCode(); } @Override @@ -63,11 +63,11 @@ protected Object parseLine(String[] row, Parser type, long sourceLine) throws Pa throw new IllegalArgumentException("Open Ranges are not allowed."); } - if(startNull){ + if (startNull) { return CDateRange.atMost(Integer.parseInt(row[endIndex])); } - if(endNull){ + if (endNull) { return CDateRange.atLeast(Integer.parseInt(row[startIndex])); } @@ -89,4 +89,4 @@ public MajorTypeId getResultType() { return new DateRangeParser(config); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/EpochOutput.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/EpochOutput.java index 510c878c36b..246df8375ba 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/EpochOutput.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/EpochOutput.java @@ -26,11 +26,8 @@ public class EpochOutput extends OutputDescription { private String inputColumn; @Override - public int hashCode(){ - return new HashCodeBuilder() - .append(super.hashCode()) - .append(inputColumn) - .toHashCode(); + public int hashCode() { + return new HashCodeBuilder().append(super.hashCode()).append(inputColumn).toHashCode(); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/LineOutput.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/LineOutput.java index d1e6961d606..51bebfea91e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/LineOutput.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/LineOutput.java @@ -17,7 +17,7 @@ */ @NoArgsConstructor(onConstructor_ = @JsonCreator) @ToString -@CPSType(id="LINE", base= OutputDescription.class) +@CPSType(id = "LINE", base = OutputDescription.class) public class LineOutput extends OutputDescription { @@ -45,6 +45,7 @@ protected Object parseLine(String[] row, Parser type, long sourceLine) throws Pa public MajorTypeId getResultType() { return MajorTypeId.INTEGER; } + @Override public Parser createParser(ConqueryConfig config) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/NullOutput.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/NullOutput.java index dc4c88663e3..eda4f52fb13 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/NullOutput.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/NullOutput.java @@ -16,17 +16,14 @@ * Output a null value. */ @Data -@CPSType(id="NULL", base= OutputDescription.class) +@CPSType(id = "NULL", base = OutputDescription.class) public class NullOutput extends OutputDescription { - + private static final long serialVersionUID = 1L; @Override - public int hashCode(){ - return new HashCodeBuilder() - .append(super.hashCode()) - .append(inputType.name()) - .toHashCode(); + public int hashCode() { + return new HashCodeBuilder().append(super.hashCode()).append(inputType.name()).toHashCode(); } @NotNull diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/OutputDescription.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/OutputDescription.java index 7738ed6273c..29d44140133 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/OutputDescription.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/OutputDescription.java @@ -1,10 +1,9 @@ package com.bakdata.conquery.models.preproc.outputs; +import jakarta.validation.constraints.NotEmpty; import java.util.InputMismatchException; import java.util.StringJoiner; -import jakarta.validation.constraints.NotEmpty; - import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.config.ConqueryConfig; @@ -58,11 +57,11 @@ public void setParent(TableInputDescriptor parent) { */ @Override public int hashCode() { - return new HashCodeBuilder() - .append(name) - .append(required) - .append(getClass().getAnnotation(CPSType.class).id()) - .toHashCode(); + return new HashCodeBuilder().append(name) + .append(required) + .append( + getClass().getAnnotation(CPSType.class).id()) + .toHashCode(); } /** @@ -91,7 +90,11 @@ public Object createOutput(String[] row, Parser type, long sourceLine) throws Pa final Object parsed = parseLine(row, type, sourceLine); if (OutputDescription.this.isRequired() && parsed == null) { - throw new IllegalArgumentException(String.format("Required Output[%s] produced NULL value at line %d", OutputDescription.this.getName(), sourceLine)); + throw new IllegalArgumentException( + String.format( + "Required Output[%s] produced NULL value at line %d", + OutputDescription.this.getName(), + sourceLine)); } return parsed; @@ -122,7 +125,8 @@ protected void assertRequiredHeaders(Object2IntArrayMap actualHeaders, S } if (missing.length() != 0) { - throw new InputMismatchException(String.format("Did not find headers `%s` in `%s`", missing.toString(), actualHeaders.keySet())); + throw new InputMismatchException( + String.format("Did not find headers `%s` in `%s`", missing.toString(), actualHeaders.keySet())); } } @@ -134,7 +138,10 @@ protected void assertRequiredHeaders(Object2IntArrayMap actualHeaders, S * @param config * @return the output for the specific headers. */ - public abstract Output createForHeaders(Object2IntArrayMap headers, DateReader dateReader, ConqueryConfig config); + public abstract Output createForHeaders( + Object2IntArrayMap headers, + DateReader dateReader, + ConqueryConfig config); /** * The resulting type after {@link Output} has been applied. diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/package-info.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/package-info.java index 5a9cdb7de7b..6a5a288ead2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/package-info.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/package-info.java @@ -8,9 +8,5 @@ *

* The file is split into three sections, so we can load them progressively: * Initially, we just read the header and determine if it isn't already loaded, and also fits to the {@link com.bakdata.conquery.models.datasets.Table} it is supposed to go in. - * We then submit an {@link com.bakdata.conquery.models.jobs.ImportJob} which will load the data. - *

- * We then load the raw data, having claims for Dictionaries in the import resolved via {@link com.bakdata.conquery.io.jackson.serializer.NsIdRef}, which is why they need to be loaded in a second step. - *

- * TODO write the rest of the documentation for {@link com.bakdata.conquery.models.jobs.ImportJob} - */ \ No newline at end of file + * We then distribute each contained {@link com.bakdata.conquery.models.events.Bucket} to a {@link com.bakdata.conquery.models.worker.Worker} on a {@link com.bakdata.conquery.commands.ShardNode}. + */ diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/ColumnValues.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/ColumnValues.java index c2857cc1716..9a1ba96e534 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/ColumnValues.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/ColumnValues.java @@ -27,8 +27,7 @@ public final int add(T value) { if (value == null) { nulls.set(event); append(nullValue); - } - else { + } else { append(value); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/Parser.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/Parser.java index f9a4addd64f..e794b76e478 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/Parser.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/Parser.java @@ -40,8 +40,7 @@ public final MAJOR_JAVA_TYPE parse(String v) throws ParsingException { } try { return parseValue(v); - } - catch (Exception e) { + } catch (Exception e) { throw new ParsingException("Failed to parse '" + v + "' with " + this.getClass().getSimpleName(), e); } } @@ -79,8 +78,7 @@ public MAJOR_JAVA_TYPE addLine(MAJOR_JAVA_TYPE v) { if (v == null) { nullLines++; - } - else { + } else { registerValue(v); } return v; diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/BooleanParser.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/BooleanParser.java index 048127c22d6..48c24345810 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/BooleanParser.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/BooleanParser.java @@ -39,7 +39,7 @@ protected BooleanStore decideType() { @Override public void setValue(BooleanStore store, int event, Boolean value) { - if(value == null){ + if (value == null) { store.setNull(event); return; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/CompoundDateRangeParser.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/CompoundDateRangeParser.java index 44cde3958d3..d02536b9698 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/CompoundDateRangeParser.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/CompoundDateRangeParser.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.preproc.parser.specific; +import jakarta.validation.constraints.NotEmpty; + import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.events.stores.primitive.BitSetStore; import com.bakdata.conquery.models.events.stores.root.DateRangeStore; @@ -8,7 +10,6 @@ import com.bakdata.conquery.models.preproc.parser.ColumnValues; import com.bakdata.conquery.models.preproc.parser.Parser; import io.dropwizard.validation.ValidationMethod; -import jakarta.validation.constraints.NotEmpty; import lombok.ToString; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateParser.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateParser.java index e3dc892c211..28b2b0075de 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateParser.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateParser.java @@ -4,7 +4,6 @@ import com.bakdata.conquery.models.common.CDate; import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.config.ParserConfig; import com.bakdata.conquery.models.events.stores.primitive.IntegerDateStore; import com.bakdata.conquery.models.events.stores.root.DateStore; import com.bakdata.conquery.models.events.stores.root.IntegerStore; @@ -41,7 +40,7 @@ protected Integer parseValue(@Nonnull String value) throws ParsingException { @Override public Integer addLine(Integer v) { - if(v == null){ + if (v == null) { subType.addLine(null); return super.addLine(null); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateRangeParser.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateRangeParser.java index 3a065e1c129..eb4dfb27337 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateRangeParser.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateRangeParser.java @@ -70,11 +70,11 @@ protected DateRangeStore decideType() { // They need to be aligned if they are non-empty. - if(!minParser.isEmpty()) { + if (!minParser.isEmpty()) { minParser.setLines(getLines()); } - if(!maxParser.isEmpty()){ + if (!maxParser.isEmpty()) { maxParser.setLines(getLines()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/IntegerParser.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/IntegerParser.java index fb5e15b0a4a..67a803f3f6a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/IntegerParser.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/IntegerParser.java @@ -55,8 +55,7 @@ protected IntegerStore decideType() { try { // avoids overflows. span = Math.subtractExact(maxValue, minValue); - } - catch (ArithmeticException exception) { + } catch (ArithmeticException exception) { return LongArrayStore.create(getLines()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/MoneyParser.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/MoneyParser.java index 31e6675686a..44ead88d28f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/MoneyParser.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/MoneyParser.java @@ -31,10 +31,10 @@ protected BigDecimal parseValue(String value) throws ParsingException { @Override protected void registerValue(BigDecimal v) { - if (maxValue == null){ + if (maxValue == null) { maxValue = v; } - if(minValue == null){ + if (minValue == null) { minValue = v; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/RealParser.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/RealParser.java index 3c53b8a1117..f1eb4cc9eba 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/RealParser.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/RealParser.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.models.preproc.parser.specific; import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.config.ParserConfig; import com.bakdata.conquery.models.events.stores.primitive.DoubleArrayStore; import com.bakdata.conquery.models.events.stores.primitive.FloatArrayStore; import com.bakdata.conquery.models.events.stores.root.RealStore; @@ -50,8 +49,7 @@ protected RealStore decideType() { if (floatULP < requiredPrecision) { return FloatArrayStore.create(getLines()); - } - else { + } else { return DoubleArrayStore.create(getLines()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/StringParser.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/StringParser.java index 7acf67e5223..8578644a929 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/StringParser.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/StringParser.java @@ -51,8 +51,6 @@ protected StringStore decideType() { } - - @Override public void setValue(StringStore store, int event, String value) { store.setString(event, value); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/C10nCache.java b/backend/src/main/java/com/bakdata/conquery/models/query/C10nCache.java index 11238e9f523..367aa0e21f4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/C10nCache.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/C10nCache.java @@ -13,6 +13,8 @@ public class C10nCache { public T getLocalized(Class clazz, Locale locale) { return (T) cache.computeIfAbsent(locale, (ignored) -> new ConcurrentHashMap<>()) - .computeIfAbsent(clazz, ignored -> C10N.get(clazz, locale)); + .computeIfAbsent( + clazz, + ignored -> C10N.get(clazz, locale)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ColumnDescriptor.java b/backend/src/main/java/com/bakdata/conquery/models/query/ColumnDescriptor.java index eca422b6712..3bd7b0999a1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ColumnDescriptor.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ColumnDescriptor.java @@ -38,7 +38,12 @@ public class ColumnDescriptor { private final Set semantics; - public ColumnDescriptor(String label, String defaultLabel, String description, String type, Set semantics){ + public ColumnDescriptor( + String label, + String defaultLabel, + String description, + String type, + Set semantics) { this.label = label; this.description = description; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/DateAggregationMode.java b/backend/src/main/java/com/bakdata/conquery/models/query/DateAggregationMode.java index d54bf4242fd..d7a2a3a6fa3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/DateAggregationMode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/DateAggregationMode.java @@ -8,21 +8,21 @@ * in certain query nodes that influence the date aggregation (see {@link CQOr} and {@link com.bakdata.conquery.models.query.queryplan.specific.OrNode} and others). */ public enum DateAggregationMode { - /** - * Don't generate an aggregated event date column at all in the result. - */ - NONE, - /** - * Merge the dates of all hit events. - */ - MERGE, - /** - * Intersect the dates of all hit events. - */ - INTERSECT, - /** - * Merge or intersect the dates depending on certain nodes in the query plan (OR -> MERGE, AND -> INTERSECT, - * NOT -> INVERT) - */ + /** + * Don't generate an aggregated event date column at all in the result. + */ + NONE, + /** + * Merge the dates of all hit events. + */ + MERGE, + /** + * Intersect the dates of all hit events. + */ + INTERSECT, + /** + * Merge or intersect the dates depending on certain nodes in the query plan (OR -> MERGE, AND -> INTERSECT, + * NOT -> INVERT) + */ LOGICAL } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/DistributedExecutionManager.java b/backend/src/main/java/com/bakdata/conquery/models/query/DistributedExecutionManager.java index de9a223b6be..73417357164 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/DistributedExecutionManager.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/DistributedExecutionManager.java @@ -43,7 +43,11 @@ public class DistributedExecutionManager extends ExecutionManager { private final ClusterState clusterState; - public DistributedExecutionManager(MetaStorage storage, DatasetRegistry datasetRegistry, ClusterState state, ConqueryConfig config) { + public DistributedExecutionManager( + MetaStorage storage, + DatasetRegistry datasetRegistry, + ClusterState state, + ConqueryConfig config) { super(storage, datasetRegistry, config); clusterState = state; } @@ -53,12 +57,16 @@ protected void doExecute(E exec log.info("Executing Query[{}] in Dataset[{}]", execution.getQueryId(), execution.getDataset()); - List resultInfo = execution instanceof SingleTableResult singleTableResult ? singleTableResult.collectResultInfos() : Collections.emptyList(); + List resultInfo = execution instanceof SingleTableResult singleTableResult ? singleTableResult + .collectResultInfos() : Collections.emptyList(); addState(execution.getId(), new DistributedExecutionInfo(resultInfo)); if (execution instanceof ManagedInternalForm form) { - form.getSubQueries().values().forEach((query) -> addState(query, new DistributedExecutionInfo(form.collectResultInfos()))); + form.getSubQueries() + .values() + .forEach( + (query) -> addState(query, new DistributedExecutionInfo(form.collectResultInfos()))); } final WorkerHandler workerHandler = getWorkerHandler(execution.getDataset()); @@ -83,7 +91,9 @@ public void doCancelQuery(ManagedExecutionId executionId) { * @implNote subQueries of Forms are managed by the form itself, so need to be passed from outside. */ @SneakyThrows - public void handleQueryResult(R result, E execution) { + public void handleQueryResult( + R result, + E execution) { if (execution == null) { log.debug("Ignoring result {} because the corresponding execution was 'null' (probably deleted)", result); @@ -101,20 +111,28 @@ public v Optional optInfo = tryGetExecutionInfo(execution.getId()); if (optInfo.isEmpty()) { - log.debug("Ignoring result {} because the corresponding state was not found (execution probably canceled)", result); + log.debug( + "Ignoring result {} because the corresponding state was not found (execution probably canceled)", + result); return; } if (!(optInfo.get() instanceof DistributedExecutionInfo distributedInfo)) { - throw new IllegalStateException("Expected execution '%s' to be of type %s, but was %s".formatted(execution.getId(), - DistributedExecutionInfo.class, - optInfo.getClass() - )); + throw new IllegalStateException( + "Expected execution '%s' to be of type %s, but was %s".formatted( + execution.getId(), + DistributedExecutionInfo.class, + optInfo.getClass() + )); } ExecutionState execState = distributedInfo.executionState; if (execState != ExecutionState.RUNNING) { - log.warn("Received result form '{}' for Query[{}] that is not RUNNING but {}", result.getWorkerId(), execution.getId(), execState); + log.warn( + "Received result form '{}' for Query[{}] that is not RUNNING but {}", + result.getWorkerId(), + execution.getId(), + execState); return; } @@ -130,7 +148,9 @@ public v // State changed to DONE or FAILED ExecutionState execStateAfterResultCollect = getExecutionInfo(execution.getId()).getExecutionState(); if (execStateAfterResultCollect != ExecutionState.RUNNING) { - final String primaryGroupName = AuthorizationHelper.getPrimaryGroup(execution.getOwner().resolve(), getStorage()).map(Group::getName).orElse("none"); + final String primaryGroupName = AuthorizationHelper.getPrimaryGroup( + execution.getOwner().resolve(), + getStorage()).map(Group::getName).orElse("none"); ExecutionMetrics.getRunningQueriesCounter(primaryGroupName).dec(); ExecutionMetrics.getQueryStateCounter(execStateAfterResultCollect, primaryGroupName).inc(); @@ -138,7 +158,10 @@ public v /* This log is here to prevent an NPE which could occur when no strong reference to result.getResults() existed anymore after the query finished and immediately was reset */ - log.trace("Collected metrics for execution {}. Last result received: {}:", result.getExecutionId(), result.getResults()); + log.trace( + "Collected metrics for execution {}. Last result received: {}:", + result.getExecutionId(), + result.getResults()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java b/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java index 662b97df284..9a69f0f5c71 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java @@ -64,8 +64,14 @@ public ExecutionManager(MetaStorage storage, DatasetRegistry datasetRegistry, this.datasetRegistry = datasetRegistry; this.config = config; - executionInfosL2 = Caffeine.from(config.getQueries().getL2CacheSpec()).removalListener(this::evictionL2).build(); - executionInfosL1 = Caffeine.from(config.getQueries().getL1CacheSpec()).removalListener(this::evictionL1).build(); + executionInfosL2 = Caffeine.from(config.getQueries().getL2CacheSpec()) + .removalListener( + this::evictionL2) + .build(); + executionInfosL1 = Caffeine.from(config.getQueries().getL1CacheSpec()) + .removalListener( + this::evictionL1) + .build(); } /** @@ -151,7 +157,11 @@ public final ManagedExecution runQuery(Namespace namespace, QueryDescription que } // Visible for testing - public final ManagedExecution createExecution(QueryDescription query, UserId user, Namespace namespace, boolean system) { + public final ManagedExecution createExecution( + QueryDescription query, + UserId user, + Namespace namespace, + boolean system) { return createExecution(query, UUID.randomUUID(), user, namespace, system); } @@ -161,13 +171,11 @@ public final void execute(ManagedExecution execution) { try { execution.initExecutable(); - } - catch (Exception e) { + } catch (Exception e) { // ConqueryErrors are usually user input errors so no need to log them at level=ERROR if (e instanceof ConqueryError) { log.warn("Failed to initialize Query[{}]", execution.getId(), e); - } - else { + } else { log.error("Failed to initialize Query[{}]", execution.getId(), e); } @@ -180,23 +188,34 @@ public final void execute(ManagedExecution execution) { execution.start(); - final String primaryGroupName = AuthorizationHelper.getPrimaryGroup(execution.getOwner().resolve(), storage).map(Group::getName).orElse("none"); + final String primaryGroupName = AuthorizationHelper.getPrimaryGroup( + execution.getOwner().resolve(), + storage).map(Group::getName).orElse("none"); ExecutionMetrics.getRunningQueriesCounter(primaryGroupName).inc(); if (execution instanceof InternalExecution internalExecution) { doExecute((ManagedExecution & InternalExecution) internalExecution); } - } - catch (Exception e) { + } catch (Exception e) { log.warn("Failed to execute '{}'", execution.getId()); execution.fail(ConqueryError.asConqueryError(e)); } } - public final ManagedExecution createExecution(QueryDescription query, UUID queryId, UserId user, Namespace namespace, boolean system) { + public final ManagedExecution createExecution( + QueryDescription query, + UUID queryId, + UserId user, + Namespace namespace, + boolean system) { // Transform the submitted query into an initialized execution - ManagedExecution managed = query.toManagedExecution(user, namespace.getDataset().getId(), storage, datasetRegistry, getConfig()); + ManagedExecution managed = query.toManagedExecution( + user, + namespace.getDataset().getId(), + storage, + datasetRegistry, + getConfig()); managed.setSystem(system); managed.setQueryId(queryId); @@ -225,8 +244,12 @@ public final void cancelExecution(final ManagedExecution execution) { public void updateState(ManagedExecutionId id, ExecutionState execState) { Optional executionInfo = tryGetExecutionInfo(id); - executionInfo.ifPresentOrElse(info -> info.setExecutionState(execState), - () -> log.warn("Could not update execution executionInfo of {} to {}, because it had no executionInfo.", id, execState) + executionInfo.ifPresentOrElse( + info -> info.setExecutionState(execState), + () -> log.warn( + "Could not update execution executionInfo of {} to {}, because it had no executionInfo.", + id, + execState) ); } @@ -280,11 +303,21 @@ public ExecutionState awaitDone(ManagedExecutionId id, int time, TimeUnit unit) } public boolean hasRunningQueries() { - if (executionInfosL2.asMap().values().stream().map(ExecutionInfo::getExecutionState).anyMatch(RUNNING::equals)) { + if (executionInfosL2.asMap() + .values() + .stream() + .map(ExecutionInfo::getExecutionState) + .anyMatch( + RUNNING::equals)) { return true; } - return executionInfosL1.asMap().values().stream().map(ExecutionInfo::getExecutionState).anyMatch(RUNNING::equals); + return executionInfosL1.asMap() + .values() + .stream() + .map(ExecutionInfo::getExecutionState) + .anyMatch( + RUNNING::equals); } /** @@ -295,7 +328,8 @@ public interface ExecutionInfo { /** * The current {@link ExecutionState} of the execution. */ - @NotNull ExecutionState getExecutionState(); + @NotNull + ExecutionState getExecutionState(); void setExecutionState(ExecutionState state); @@ -309,7 +343,9 @@ public interface ExecutionInfo { public interface InternalExecutionInfo extends ExecutionInfo { Stream streamQueryResults(); + List getResultInfos(); + long getResultCount(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ExternalExecutionInfoImpl.java b/backend/src/main/java/com/bakdata/conquery/models/query/ExternalExecutionInfoImpl.java index 9c145bd42cd..0beec0d8dfa 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ExternalExecutionInfoImpl.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ExternalExecutionInfoImpl.java @@ -1,10 +1,10 @@ package com.bakdata.conquery.models.query; +import jakarta.ws.rs.core.Response; import java.util.Collections; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.stream.Stream; -import jakarta.ws.rs.core.Response; import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.io.external.form.ExternalFormBackendApi; @@ -54,8 +54,10 @@ public Stream getResultAssets() { @Override public Response fetchExternalResult(String assetId) { final ResultAsset resultRef = resultsAssetMap.stream() - .map(Pair::key).filter(a -> a.getAssetId().equals(assetId)) - .collect(MoreCollectors.onlyElement()); + .map(Pair::key) + .filter( + a -> a.getAssetId().equals(assetId)) + .collect(MoreCollectors.onlyElement()); return api.getResult(resultRef.url()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java index 830f0a2c558..de9489a61fb 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java @@ -1,12 +1,12 @@ package com.bakdata.conquery.models.query; -import javax.validation.constraints.NotNull; import java.util.List; import java.util.Optional; import java.util.OptionalLong; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; import java.util.stream.Stream; +import javax.validation.constraints.NotNull; import com.bakdata.conquery.apiv1.execution.ExecutionStatus; import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; @@ -55,7 +55,13 @@ public class ManagedQuery extends ManagedExecution implements SingleTableResult, @NotNull private Query query; - public ManagedQuery(Query query, UserId owner, DatasetId submittedDataset, MetaStorage storage, DatasetRegistry datasetRegistry, ConqueryConfig config) { + public ManagedQuery( + Query query, + UserId owner, + DatasetId submittedDataset, + MetaStorage storage, + DatasetRegistry datasetRegistry, + ConqueryConfig config) { super(owner, submittedDataset, storage, datasetRegistry, config); this.query = query; } @@ -74,7 +80,7 @@ public synchronized void finish(ExecutionState executionState) { public Stream streamResults(OptionalLong maybeLimit) { final Stream results = getNamespace().getExecutionManager().streamQueryResults(this); - if(maybeLimit.isEmpty()){ + if (maybeLimit.isEmpty()) { return results; } @@ -89,10 +95,10 @@ public synchronized OptionalLong resultRowCount() { ExecutionManager executionManager = getExecutionManager(); Optional executionInfo = executionManager.tryGetExecutionInfo(getId()); - return executionInfo - .map(ExecutionManager.InternalExecutionInfo::getResultCount) - .map(OptionalLong::of) - .orElse(OptionalLong.empty()); + return executionInfo.map(ExecutionManager.InternalExecutionInfo::getResultCount) + .map(OptionalLong::of) + .orElse( + OptionalLong.empty()); } @Override @@ -128,7 +134,9 @@ public List collectResultInfos() { @Override @JsonIgnore public List getResultInfos() { - ExecutionManager.InternalExecutionInfo executionInfo = getNamespace().getExecutionManager().getExecutionInfo(getId()); + ExecutionManager.InternalExecutionInfo executionInfo = getNamespace().getExecutionManager() + .getExecutionInfo( + getId()); return executionInfo.getResultInfos(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java b/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java index 9ee40c30cca..e899e75612b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java @@ -51,11 +51,30 @@ public class PrintSettings { private final PrintIdMapper idMapper; - public PrintSettings(boolean prettyPrint, Locale locale, ConqueryConfig config, PrintIdMapper idMapper, Function columnNamer) { - this(prettyPrint, locale, config, idMapper, columnNamer, DECIMAL_FORMAT.apply(locale), NUMBER_FORMAT.apply(locale)); + public PrintSettings( + boolean prettyPrint, + Locale locale, + ConqueryConfig config, + PrintIdMapper idMapper, + Function columnNamer) { + this( + prettyPrint, + locale, + config, + idMapper, + columnNamer, + DECIMAL_FORMAT.apply(locale), + NUMBER_FORMAT.apply(locale)); } - public PrintSettings(boolean prettyPrint, Locale locale, ConqueryConfig config, PrintIdMapper idMapper, Function columnNamer, NumberFormat decimalFormat, NumberFormat numberFormat) { + public PrintSettings( + boolean prettyPrint, + Locale locale, + ConqueryConfig config, + PrintIdMapper idMapper, + Function columnNamer, + NumberFormat decimalFormat, + NumberFormat numberFormat) { this.prettyPrint = prettyPrint; this.locale = locale; this.currency = config.getPreprocessor().getParsers().getCurrency(); @@ -66,7 +85,9 @@ public PrintSettings(boolean prettyPrint, Locale locale, ConqueryConfig config, this.decimalFormat = decimalFormat; this.listFormat = prettyPrint ? config.getLocale().getListFormats().getFirst() : UNPRETTY_LIST_FORMAT; - this.dateRangeSeparator = prettyPrint ? config.getLocale().findDateRangeSeparator(locale) : UNPRETTY_DATERANGE_SEPERATOR; + this.dateRangeSeparator = prettyPrint ? config.getLocale() + .findDateRangeSeparator( + locale) : UNPRETTY_DATERANGE_SEPERATOR; this.dateFormat = config.getLocale().findDateFormat(locale); this.dateFormatter = prettyPrint ? DateTimeFormatter.ofPattern(dateFormat) : UNPRETTY_DATEFORMATTER; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/QueryExecutionContext.java b/backend/src/main/java/com/bakdata/conquery/models/query/QueryExecutionContext.java index 8bdc179913e..d08bbe93a3e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/QueryExecutionContext.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/QueryExecutionContext.java @@ -1,73 +1,68 @@ -package com.bakdata.conquery.models.query; - -import java.time.Clock; -import java.time.LocalDate; -import java.time.temporal.TemporalField; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.Set; - -import com.bakdata.conquery.apiv1.query.concept.specific.CQTemporal; -import com.bakdata.conquery.io.storage.ModificationShieldedWorkerStorage; -import com.bakdata.conquery.models.common.CDate; -import com.bakdata.conquery.models.common.CDateSet; -import com.bakdata.conquery.models.datasets.SecondaryIdDescription; -import com.bakdata.conquery.models.datasets.concepts.Connector; -import com.bakdata.conquery.models.datasets.concepts.ValidityDate; -import com.bakdata.conquery.models.events.BucketManager; -import com.bakdata.conquery.models.identifiable.ids.specific.BucketId; -import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; -import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId; -import com.bakdata.conquery.models.identifiable.ids.specific.TableId; -import com.bakdata.conquery.models.query.entity.Entity; -import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import lombok.With; - -@Data -@AllArgsConstructor -@RequiredArgsConstructor -@With -public class QueryExecutionContext { - - private final ManagedExecutionId executionId; - private final QueryExecutor executor; - private final ModificationShieldedWorkerStorage storage; - private final BucketManager bucketManager; - private final Clock clock; - - - public int getToday() { - return CDate.ofLocalDate(LocalDate.now(clock)); - } - - private ValidityDate validityDateColumn; - - private Connector connector; - - @NonNull - private CDateSet dateRestriction = CDateSet.createFull(); - @NonNull - private Optional> queryDateAggregator = Optional.empty(); - - - /** - * Only set when in {@link com.bakdata.conquery.models.query.queryplan.SecondaryIdQueryPlan}, to the selected {@link SecondaryIdDescriptionId}. - */ - private SecondaryIdDescription activeSecondaryId = null; - - - - public Set getEntityBucketsForTable(Entity entity, TableId table) { - return bucketManager.getEntityBucketsForTable(entity, table); - } - - boolean isQueryCancelled() { - return executor.isCancelled(executionId); - } - -} \ No newline at end of file +package com.bakdata.conquery.models.query; + +import java.time.Clock; +import java.time.LocalDate; +import java.util.Optional; +import java.util.Set; + +import com.bakdata.conquery.io.storage.ModificationShieldedWorkerStorage; +import com.bakdata.conquery.models.common.CDate; +import com.bakdata.conquery.models.common.CDateSet; +import com.bakdata.conquery.models.datasets.SecondaryIdDescription; +import com.bakdata.conquery.models.datasets.concepts.Connector; +import com.bakdata.conquery.models.datasets.concepts.ValidityDate; +import com.bakdata.conquery.models.events.BucketManager; +import com.bakdata.conquery.models.identifiable.ids.specific.BucketId; +import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; +import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId; +import com.bakdata.conquery.models.identifiable.ids.specific.TableId; +import com.bakdata.conquery.models.query.entity.Entity; +import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.With; + +@Data +@AllArgsConstructor +@RequiredArgsConstructor +@With +public class QueryExecutionContext { + + private final ManagedExecutionId executionId; + private final QueryExecutor executor; + private final ModificationShieldedWorkerStorage storage; + private final BucketManager bucketManager; + private final Clock clock; + + + public int getToday() { + return CDate.ofLocalDate(LocalDate.now(clock)); + } + + private ValidityDate validityDateColumn; + + private Connector connector; + + @NonNull + private CDateSet dateRestriction = CDateSet.createFull(); + @NonNull + private Optional> queryDateAggregator = Optional.empty(); + + + /** + * Only set when in {@link com.bakdata.conquery.models.query.queryplan.SecondaryIdQueryPlan}, to the selected {@link SecondaryIdDescriptionId}. + */ + private SecondaryIdDescription activeSecondaryId = null; + + + public Set getEntityBucketsForTable(Entity entity, TableId table) { + return bucketManager.getEntityBucketsForTable(entity, table); + } + + boolean isQueryCancelled() { + return executor.isCancelled(executionId); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/QueryExecutor.java b/backend/src/main/java/com/bakdata/conquery/models/query/QueryExecutor.java index 5c8690a78ce..2301ff1f1c1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/QueryExecutor.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/QueryExecutor.java @@ -57,13 +57,17 @@ public boolean isCancelled(ManagedExecutionId query) { return cancelledQueries.contains(query); } - public boolean execute(Query query, QueryExecutionContext executionContext, ShardResult result, Set entities) { + public boolean execute( + Query query, + QueryExecutionContext executionContext, + ShardResult result, + Set entities) { log.info("Received query: {}", query); Stopwatch stopwatch = Stopwatch.createStarted(); - final ThreadLocal> plan = - ThreadLocal.withInitial(() -> query.createQueryPlan(new QueryPlanContext(executionContext.getStorage(), secondaryIdSubPlanLimit))); + final ThreadLocal> plan = ThreadLocal.withInitial( + () -> query.createQueryPlan(new QueryPlanContext(executionContext.getStorage(), secondaryIdSubPlanLimit))); log.trace("Created query plan in {}", stopwatch); if (entities.isEmpty()) { @@ -77,8 +81,12 @@ public boolean execute(Query query, QueryExecutionContext executionContext, Shar log.debug("QueryPlan for Query[{}] = `{}`", result.getExecutionId(), plan.get()); } - final List>> futures = - entities.stream().map(entity -> new QueryJob(executionContext, plan, entity)).map(job -> CompletableFuture.supplyAsync(job, executor)).toList(); + final List>> futures = entities.stream() + .map( + entity -> new QueryJob(executionContext, plan, entity)) + .map( + job -> CompletableFuture.supplyAsync(job, executor)) + .toList(); final CompletableFuture allDone = CompletableFuture.allOf(futures.toArray(CompletableFuture[]::new)); @@ -87,7 +95,12 @@ public boolean execute(Query query, QueryExecutionContext executionContext, Shar allDone.join(); // We are in the clear here, all futures/entities completed successfully - List entityResults = futures.stream().map(CompletableFuture::join).filter(Optional::isPresent).map(Optional::get).toList(); + List entityResults = futures.stream() + .map(CompletableFuture::join) + .filter( + Optional::isPresent) + .map(Optional::get) + .toList(); // Prepare message to manager result.finish(entityResults, worker); @@ -95,10 +108,9 @@ public boolean execute(Query query, QueryExecutionContext executionContext, Shar sendResultToManagerNode(result); return true; - } - catch (Throwable e) { + } catch (Throwable e) { log.warn("Error while executing {}", executionContext.getExecutionId(), e); - sendFailureToManagerNode(asConqueryError(e),executionContext.getExecutionId()); + sendFailureToManagerNode(asConqueryError(e), executionContext.getExecutionId()); return false; } } @@ -126,8 +138,7 @@ private void sendResultToManagerNode(ShardResult result) { sendFailureToManagerNode(exception, result.getExecutionId()); }); - } - catch (OutOfMemoryError oome) { + } catch (OutOfMemoryError oome) { // Result was too large for serialization throw new ConqueryError.ExecutionProcessingResultSizeError(); } @@ -149,9 +160,10 @@ private static IoFutureListener logFailureTransmission(ManagedExecutio log.info("Successfully informed manager about failed shard result for execution {}", executionId); return; } - log.error("Could not notify manager about shard result submission failure for execution {}. " - + "Manager probably has this execution in a dangling {} state", - executionId, ExecutionState.RUNNING + log.error( + "Could not notify manager about shard result submission failure for execution {}. " + "Manager probably has this execution in a dangling {} state", + executionId, + ExecutionState.RUNNING ); }; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/QueryJob.java b/backend/src/main/java/com/bakdata/conquery/models/query/QueryJob.java index f7212885168..827d1f4270a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/QueryJob.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/QueryJob.java @@ -28,12 +28,10 @@ public Optional get() { queryPlan.init(ctx, entity); return queryPlan.execute(ctx, entity); - } - catch (ConqueryError e) { + } catch (ConqueryError e) { // Catch errors, propagate them with their id. throw new ConqueryError.ExecutionJobErrorWrapper(entity, e); - } - catch (Exception e) { + } catch (Exception e) { throw new ConqueryError.ExecutionJobErrorWrapper(entity, new ConqueryError.UnknownError(e)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/QueryPlanContext.java b/backend/src/main/java/com/bakdata/conquery/models/query/QueryPlanContext.java index 3a10e392704..8692c3407ec 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/QueryPlanContext.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/QueryPlanContext.java @@ -1,41 +1,41 @@ -package com.bakdata.conquery.models.query; - -import javax.annotation.Nullable; - -import com.bakdata.conquery.io.storage.WorkerStorage; -import com.bakdata.conquery.models.common.daterange.CDateRange; -import com.bakdata.conquery.models.datasets.Dataset; -import com.bakdata.conquery.models.datasets.SecondaryIdDescription; -import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.RequiredArgsConstructor; -import lombok.With; - -@Data -@With -@AllArgsConstructor -@RequiredArgsConstructor -public class QueryPlanContext { - - private final WorkerStorage storage; - private final int secondaryIdSubPlanRetention; - - private CDateRange dateRestriction = CDateRange.all(); - - private boolean disableAggregators = false; - private boolean disableAggregationFilters = false; - - - /** - * Set if in {@link com.bakdata.conquery.models.query.queryplan.SecondaryIdQueryPlan}, to the query-active {@link SecondaryIdDescriptionId}. - */ - @Nullable - private SecondaryIdDescription selectedSecondaryId; - - public Dataset getDataset() { - return getStorage().getDataset(); - } - - -} +package com.bakdata.conquery.models.query; + +import javax.annotation.Nullable; + +import com.bakdata.conquery.io.storage.WorkerStorage; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.datasets.SecondaryIdDescription; +import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.RequiredArgsConstructor; +import lombok.With; + +@Data +@With +@AllArgsConstructor +@RequiredArgsConstructor +public class QueryPlanContext { + + private final WorkerStorage storage; + private final int secondaryIdSubPlanRetention; + + private CDateRange dateRestriction = CDateRange.all(); + + private boolean disableAggregators = false; + private boolean disableAggregationFilters = false; + + + /** + * Set if in {@link com.bakdata.conquery.models.query.queryplan.SecondaryIdQueryPlan}, to the query-active {@link SecondaryIdDescriptionId}. + */ + @Nullable + private SecondaryIdDescription selectedSecondaryId; + + public Dataset getDataset() { + return getStorage().getDataset(); + } + + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/RequiredEntities.java b/backend/src/main/java/com/bakdata/conquery/models/query/RequiredEntities.java index 6d44d86cf4a..37f9e1ce87b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/RequiredEntities.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/RequiredEntities.java @@ -43,9 +43,9 @@ public RequiredEntities union(@NonNull RequiredEntities other) { public Set resolve(BucketManager bucketManager) { final Set all = bucketManager.getEntities(); return entities.stream() - .filter(all::contains) - // The following is just a wrapping, that is later unwrapped again in an execution - .map(Entity::new) - .collect(Collectors.toSet()); + .filter(all::contains) + // The following is just a wrapping, that is later unwrapped again in an execution + .map(Entity::new) + .collect(Collectors.toSet()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/SingleTableResult.java b/backend/src/main/java/com/bakdata/conquery/models/query/SingleTableResult.java index 39fd4136d6e..fbfb35d184f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/SingleTableResult.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/SingleTableResult.java @@ -30,8 +30,12 @@ default List generateColumnDescriptions(boolean isInitialized, // First add the id columns to the descriptor list. The are the first columns for (ResultInfo header : config.getIdColumns().getIdResultInfos()) { - final ColumnDescriptor descriptor = - new ColumnDescriptor(uniqNamer.getUniqueName(header, settings), null, null, ResultType.Primitive.STRING.typeInfo(), header.getSemantics()); + final ColumnDescriptor descriptor = new ColumnDescriptor( + uniqNamer.getUniqueName(header, settings), + null, + null, + ResultType.Primitive.STRING.typeInfo(), + header.getSemantics()); columnDescriptions.add(descriptor); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/Visitable.java b/backend/src/main/java/com/bakdata/conquery/models/query/Visitable.java index 2af0b5682d2..3846cd0cbaf 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/Visitable.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/Visitable.java @@ -10,12 +10,12 @@ public interface Visitable { /** * Creates a stream of all elements in the Visitable. */ - public static Stream stream(Visitable visitable) { + static Stream stream(Visitable visitable) { final Stream.Builder builder = Stream.builder(); visitable.visit(builder); return builder.build(); } - + } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/DateDistanceFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/DateDistanceFilterNode.java index 77693b48115..21431bbb96a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/DateDistanceFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/DateDistanceFilterNode.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.query.filter.event; +import jakarta.validation.constraints.NotNull; import java.time.LocalDate; import java.time.temporal.ChronoUnit; import java.util.Set; @@ -11,7 +12,6 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.queryplan.filter.EventFilterNode; -import jakarta.validation.constraints.NotNull; import lombok.Getter; import lombok.Setter; import lombok.ToString; @@ -40,8 +40,7 @@ public DateDistanceFilterNode(Column column, ChronoUnit unit, Range.LongRange fi public void nextTable(QueryExecutionContext ctx, Table currentTable) { if (CDate.isPositiveInfinity(ctx.getDateRestriction().getMaxValue()) || ctx.getDateRestriction().isEmpty()) { reference = LocalDate.now(); - } - else { + } else { reference = CDate.toLocalDate(ctx.getDateRestriction().getMaxValue()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/MultiSelectFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/MultiSelectFilterNode.java index ca45918a8be..ba393e242e3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/MultiSelectFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/MultiSelectFilterNode.java @@ -1,7 +1,7 @@ package com.bakdata.conquery.models.query.filter.event; -import java.util.Set; import jakarta.validation.constraints.NotNull; +import java.util.Set; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.Table; @@ -27,7 +27,7 @@ public class MultiSelectFilterNode extends EventFilterNode> { private Column column; private final boolean empty; - + public MultiSelectFilterNode(Column column, Set filterValue) { super(filterValue); @@ -65,4 +65,4 @@ public boolean isOfInterest(Bucket bucket) { public void collectRequiredTables(Set

requiredTables) { requiredTables.add(column.getTable()); } - } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/PrefixTextFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/PrefixTextFilterNode.java index 23ca0ca2438..9293db01cba 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/PrefixTextFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/PrefixTextFilterNode.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.query.filter.event; +import jakarta.validation.constraints.NotNull; import java.util.Set; import com.bakdata.conquery.models.datasets.Column; @@ -7,7 +8,6 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.events.stores.root.StringStore; import com.bakdata.conquery.models.query.queryplan.filter.EventFilterNode; -import jakarta.validation.constraints.NotNull; import lombok.Getter; import lombok.Setter; import lombok.ToString; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/SubstringMultiSelectFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/SubstringMultiSelectFilterNode.java index 43c7d5827d6..3eb50432568 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/SubstringMultiSelectFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/SubstringMultiSelectFilterNode.java @@ -1,7 +1,7 @@ package com.bakdata.conquery.models.query.filter.event; -import java.util.Set; import jakarta.validation.constraints.NotNull; +import java.util.Set; import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.datasets.Column; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/number/NumberFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/number/NumberFilterNode.java index 3059a9c2989..7c1643f8b57 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/number/NumberFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/number/NumberFilterNode.java @@ -1,8 +1,7 @@ package com.bakdata.conquery.models.query.filter.event.number; -import java.util.Set; - import jakarta.validation.constraints.NotNull; +import java.util.Set; import com.bakdata.conquery.models.common.IRange; import com.bakdata.conquery.models.datasets.Column; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java index 8a10f25ecc3..4f974e34787 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java @@ -63,12 +63,12 @@ public class EntityPreviewExecution extends ManagedInternalForm datasetRegistry, - ConqueryConfig config) { + EntityPreviewForm entityPreviewQuery, + UserId user, + DatasetId submittedDataset, + MetaStorage storage, + DatasetRegistry datasetRegistry, + ConqueryConfig config) { super(entityPreviewQuery, user, submittedDataset, storage, datasetRegistry, config); } @@ -89,17 +89,21 @@ public List generateColumnDescriptions(boolean isInitialized, for (ColumnDescriptor descriptor : descriptors) { // Add grouping semantics to secondaryIds to group by if (descriptor.getSemantics() - .stream() - .anyMatch(semanticType -> semanticType instanceof SemanticType.SecondaryIdT desc - && previewConfig.isGroupingColumn(desc.getSecondaryId()) - )) { + .stream() + .anyMatch( + semanticType -> semanticType instanceof SemanticType.SecondaryIdT desc && previewConfig + .isGroupingColumn( + desc.getSecondaryId()) + )) { descriptor.getSemantics().add(new SemanticType.GroupT()); } // Add hidden semantics to fields flagged for hiding. if (descriptor.getSemantics() - .stream() - .anyMatch(semanticType -> semanticType instanceof SemanticType.ColumnT desc && previewConfig.isHidden(desc.getColumn()))) { + .stream() + .anyMatch( + semanticType -> semanticType instanceof SemanticType.ColumnT desc && previewConfig.isHidden( + desc.getColumn()))) { descriptor.getSemantics().add(new SemanticType.HiddenT()); } } @@ -136,10 +140,20 @@ public FullExecutionStatus buildStatusFull(Subject subject, Namespace namespace) JsonResultPrinters printers = new JsonResultPrinters(); - final PrintSettings infoSettings = new PrintSettings(true, I18n.LOCALE.get(), getConfig(), null, previewConfig::resolveSelectLabel); + final PrintSettings infoSettings = new PrintSettings( + true, + I18n.LOCALE.get(), + getConfig(), + null, + previewConfig::resolveSelectLabel); status.setInfos(transformQueryResultToInfos(getInfoCardExecution(), infoSettings, printers)); - final PrintSettings stratifiedSettings = new PrintSettings(false, I18n.LOCALE.get(), getConfig(), null, previewConfig::resolveSelectLabel); + final PrintSettings stratifiedSettings = new PrintSettings( + false, + I18n.LOCALE.get(), + getConfig(), + null, + previewConfig::resolveSelectLabel); status.setTimeStratifiedInfos(toChronoInfos(previewConfig, resolvedSubQueries(), stratifiedSettings, printers)); return status; @@ -150,11 +164,14 @@ public FullExecutionStatus buildStatusFull(Subject subject, Namespace namespace) * The format of the query is an {@link AbsoluteFormQuery} containing a single line for one person. This should correspond to {@link EntityPreviewForm#VALUES_QUERY_NAME}. */ private List transformQueryResultToInfos( - ManagedQuery infoCardExecution, PrintSettings printSettings, PrinterFactory printerFactory) { + ManagedQuery infoCardExecution, + PrintSettings printSettings, + PrinterFactory printerFactory) { // Submitted Query is a single line of an AbsoluteFormQuery => MultilineEntityResult with a single line. - final MultilineEntityResult result = (MultilineEntityResult) infoCardExecution.streamResults(OptionalLong.empty()).collect(MoreCollectors.onlyElement()); + final MultilineEntityResult result = (MultilineEntityResult) infoCardExecution.streamResults( + OptionalLong.empty()).collect(MoreCollectors.onlyElement()); final Object[] values = result.getValues().getFirst(); final List extraInfos = new ArrayList<>(values.length); @@ -169,18 +186,19 @@ private List transformQueryResultToInfos( if (value == null) { printed = null; - } - else { + } else { Printer printer = resultInfo.createPrinter(printerFactory, printSettings); printed = printer.apply(value); } - extraInfos.add(new EntityPreviewStatus.Info(resultInfo.userColumnName(printSettings), - printed, - resultInfo.getType().typeInfo(), - resultInfo.getDescription(), - resultInfo.getSemantics() - )); + extraInfos.add( + new EntityPreviewStatus.Info( + resultInfo.userColumnName(printSettings), + printed, + resultInfo.getType().typeInfo(), + resultInfo.getDescription(), + resultInfo.getSemantics() + )); } return extraInfos; @@ -194,20 +212,31 @@ private ManagedQuery getInfoCardExecution() { @NotNull private List toChronoInfos( - PreviewConfig previewConfig, Map subQueries, PrintSettings printSettings, PrinterFactory printers) { + PreviewConfig previewConfig, + Map subQueries, + PrintSettings printSettings, + PrinterFactory printers) { final List timeStratifiedInfos = new ArrayList<>(); for (PreviewConfig.TimeStratifiedSelects description : previewConfig.getTimeStratifiedSelects()) { final ManagedQuery query = subQueries.get(description.label()); query.initExecutable(); - final EntityResult entityResult = query.streamResults(OptionalLong.empty()).collect(MoreCollectors.onlyElement()); + final EntityResult entityResult = query.streamResults(OptionalLong.empty()) + .collect( + MoreCollectors.onlyElement()); - final Map select2desc = - description.selects().stream().collect(Collectors.toMap(PreviewConfig.InfoCardSelect::select, Function.identity())); + final Map select2desc = description.selects() + .stream() + .collect( + Collectors.toMap(PreviewConfig.InfoCardSelect::select, Function.identity())); // Group lines by year and quarter. - final Function> lineTransformer = createLineToMapTransformer(query.collectResultInfos(), select2desc, printSettings, printers); + final Function> lineTransformer = createLineToMapTransformer( + query.collectResultInfos(), + select2desc, + printSettings, + printers); final List yearEntries = createYearEntries(entityResult, lineTransformer); final Object[] completeResult = getCompleteLine(entityResult); @@ -216,13 +245,13 @@ private List toChronoInfos( final List columnDescriptors = createChronoColumnDescriptors(query, select2desc); - final EntityPreviewStatus.TimeStratifiedInfos infos = - new EntityPreviewStatus.TimeStratifiedInfos(description.label(), - description.description(), - columnDescriptors, - lineTransformer.apply(completeResult), - yearEntries - ); + final EntityPreviewStatus.TimeStratifiedInfos infos = new EntityPreviewStatus.TimeStratifiedInfos( + description.label(), + description.description(), + columnDescriptors, + lineTransformer.apply(completeResult), + yearEntries + ); timeStratifiedInfos.add(infos); } @@ -235,7 +264,10 @@ private List toChronoInfos( * Null values are omitted. */ private static Function> createLineToMapTransformer( - List resultInfos, Map select2desc, PrintSettings printSettings, PrinterFactory printerFactory) { + List resultInfos, + Map select2desc, + PrintSettings printSettings, + PrinterFactory printerFactory) { final int size = resultInfos.size(); @@ -276,7 +308,9 @@ private static Function> createLineToMapTransforme } @NotNull - private List createYearEntries(EntityResult entityResult, Function> lineTransformer) { + private List createYearEntries( + EntityResult entityResult, + Function> lineTransformer) { final Map yearLines = getYearLines(entityResult); final Map> quarterLines = getQuarterLines(entityResult); @@ -286,7 +320,9 @@ private List createYearEntries(EntityResult entit final List quarterEntries = new ArrayList<>(); quarterLines.getOrDefault(year, Collections.emptyMap()) - .forEach((quarter, line) -> quarterEntries.add(new EntityPreviewStatus.QuarterEntry(quarter, lineTransformer.apply(line)))); + .forEach( + (quarter, line) -> quarterEntries.add( + new EntityPreviewStatus.QuarterEntry(quarter, lineTransformer.apply(line)))); quarterEntries.sort(Comparator.comparingInt(EntityPreviewStatus.QuarterEntry::quarter)); @@ -312,7 +348,9 @@ private Object[] getCompleteLine(EntityResult entityResult) { /** * For the selects in result infos, build ColumnDescriptors using definitions (label and description) from PreviewConfig. */ - private static List createChronoColumnDescriptors(SingleTableResult query, Map select2desc) { + private static List createChronoColumnDescriptors( + SingleTableResult query, + Map select2desc) { final List columnDescriptions = new ArrayList<>(); @@ -321,12 +359,14 @@ private static List createChronoColumnDescriptors(SingleTableR final PreviewConfig.InfoCardSelect desc = select2desc.get(selectResultInfo.getSelect().getId()); // We build these by hand because they are labeled and described by config. - columnDescriptions.add(new ColumnDescriptor(desc.label(), - desc.label(), - desc.description() != null ? desc.description() : selectResultInfo.getDescription(), // both might be null - info.getType().typeInfo(), - info.getSemantics() - )); + columnDescriptions.add( + new ColumnDescriptor( + desc.label(), + desc.label(), + desc.description() != null ? desc.description() : selectResultInfo.getDescription(), // both might be null + info.getType().typeInfo(), + info.getSemantics() + )); } } @@ -382,7 +422,10 @@ private static Map> getQuarterLines(EntityResult } @Override - protected void setAdditionalFieldsForStatusWithSource(Subject subject, FullExecutionStatus status, Namespace namespace) { + protected void setAdditionalFieldsForStatusWithSource( + Subject subject, + FullExecutionStatus status, + Namespace namespace) { status.setColumnDescriptions(generateColumnDescriptions(isInitialized(), getConfig())); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewForm.java b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewForm.java index 80361b9783a..6d5714a25e9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewForm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewForm.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.query.preview; +import jakarta.validation.Valid; import java.time.LocalDate; import java.util.HashMap; import java.util.HashSet; @@ -8,7 +9,6 @@ import java.util.Set; import java.util.function.Consumer; import java.util.stream.Collectors; -import jakarta.validation.Valid; import com.bakdata.conquery.apiv1.forms.Form; import com.bakdata.conquery.apiv1.forms.InternalForm; @@ -79,28 +79,36 @@ public class EntityPreviewForm extends Form implements InternalForm { private final Map timeOverViews; public static EntityPreviewForm create( - String entity, - String idKind, - Range dateRange, - List sources, - List infos, - List timeStratifiedSelects, - DatasetRegistry datasetRegistry) { + String entity, + String idKind, + Range dateRange, + List sources, + List infos, + List timeStratifiedSelects, + DatasetRegistry datasetRegistry) { // We use this query to filter for the single selected query. - final Query entitySelectQuery = new ConceptQuery(new CQExternal(List.of(idKind), new String[][]{{"HEAD"}, {entity}}, true)); + final Query entitySelectQuery = new ConceptQuery( + new CQExternal(List.of(idKind), new String[][]{{"HEAD"}, {entity}}, true)); final TableExportQuery exportQuery = createExportQuery(dateRange, sources, entitySelectQuery); final AbsoluteFormQuery infoCardQuery = createInfoCardQuery(dateRange, infos, entitySelectQuery); - final Map timeQueries = createTimeStratifiedQueries(dateRange, timeStratifiedSelects, datasetRegistry, entitySelectQuery); + final Map timeQueries = createTimeStratifiedQueries( + dateRange, + timeStratifiedSelects, + datasetRegistry, + entitySelectQuery); return new EntityPreviewForm(infoCardQuery, exportQuery, timeQueries); } @NotNull - private static TableExportQuery createExportQuery(Range dateRange, List sources, Query entitySelectQuery) { + private static TableExportQuery createExportQuery( + Range dateRange, + List sources, + Query entitySelectQuery) { // Query exporting selected Sources of the Entity. final TableExportQuery exportQuery = new TableExportQuery(entitySelectQuery); @@ -111,36 +119,47 @@ private static TableExportQuery createExportQuery(Range dateRange, Li } @NotNull - private static AbsoluteFormQuery createInfoCardQuery(Range dateRange, List infos, Query entitySelectQuery) { + private static AbsoluteFormQuery createInfoCardQuery( + Range dateRange, + List infos, + Query entitySelectQuery) { // Query exporting a few additional infos on the entity. - return new AbsoluteFormQuery(entitySelectQuery, dateRange, - ArrayConceptQuery.createFromFeatures( - infos.stream() - .map(CQConcept::forSelect) - .collect(Collectors.toList()) - ), - List.of(ExportForm.ResolutionAndAlignment.of(Resolution.COMPLETE, Alignment.NO_ALIGN)) + return new AbsoluteFormQuery( + entitySelectQuery, + dateRange, + ArrayConceptQuery.createFromFeatures( + infos.stream().map(CQConcept::forSelect).collect(Collectors.toList()) + ), + List.of(ExportForm.ResolutionAndAlignment.of(Resolution.COMPLETE, Alignment.NO_ALIGN)) ); } @NotNull - private static Map createTimeStratifiedQueries(Range dateRange, List timeStratifiedSelects, DatasetRegistry datasetRegistry, Query entitySelectQuery) { + private static Map createTimeStratifiedQueries( + Range dateRange, + List timeStratifiedSelects, + DatasetRegistry datasetRegistry, + Query entitySelectQuery) { final Map timeQueries = new HashMap<>(); // per group create an AbsoluteFormQuery on years and quarters. for (PreviewConfig.TimeStratifiedSelects selects : timeStratifiedSelects) { - final AbsoluteFormQuery query = new AbsoluteFormQuery(entitySelectQuery, dateRange, - ArrayConceptQuery.createFromFeatures( - selects.selects().stream() - .map(PreviewConfig.InfoCardSelect::select) - .map(CQConcept::forSelect) - .collect(Collectors.toList())), - List.of( - ExportForm.ResolutionAndAlignment.of(Resolution.COMPLETE, Alignment.NO_ALIGN), - ExportForm.ResolutionAndAlignment.of(Resolution.YEARS, Alignment.YEAR), - ExportForm.ResolutionAndAlignment.of(Resolution.QUARTERS, Alignment.QUARTER) - ) + final AbsoluteFormQuery query = new AbsoluteFormQuery( + entitySelectQuery, + dateRange, + ArrayConceptQuery.createFromFeatures( + selects.selects() + .stream() + .map(PreviewConfig.InfoCardSelect::select) + .map( + CQConcept::forSelect) + .collect(Collectors.toList())), + List.of( + ExportForm.ResolutionAndAlignment.of(Resolution.COMPLETE, Alignment.NO_ALIGN), + ExportForm.ResolutionAndAlignment.of(Resolution.YEARS, Alignment.YEAR), + ExportForm.ResolutionAndAlignment.of(Resolution.QUARTERS, Alignment.QUARTER) + ) ); timeQueries.put(selects.label(), query); @@ -156,7 +175,11 @@ public JsonNode getValues() { } @Override - public void authorize(Subject subject, DatasetId submittedDataset, @NonNull List visitors, MetaStorage storage) { + public void authorize( + Subject subject, + DatasetId submittedDataset, + @NonNull List visitors, + MetaStorage storage) { QueryDescription.authorizeQuery(this, subject, submittedDataset, visitors, storage); } @@ -179,7 +202,12 @@ public Map createSubQueries() { } @Override - public ManagedExecution toManagedExecution(UserId user, DatasetId submittedDataset, MetaStorage storage, DatasetRegistry datasetRegistry, ConqueryConfig config) { + public ManagedExecution toManagedExecution( + UserId user, + DatasetId submittedDataset, + MetaStorage storage, + DatasetRegistry datasetRegistry, + ConqueryConfig config) { return new EntityPreviewExecution(this, user, submittedDataset, storage, datasetRegistry, config); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewStatus.java b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewStatus.java index 34ffe0625c8..4af3f27bd41 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewStatus.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewStatus.java @@ -3,7 +3,6 @@ import java.util.List; import java.util.Map; import java.util.Set; - import javax.annotation.Nullable; import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; @@ -46,7 +45,12 @@ public Info(String label, Object value, String typeInfo, String description, Set * @param columns Typed description of the infos. * @param years Structured results, also contains QuarterEntries. */ - public record TimeStratifiedInfos(String label, String description, List columns, Map totals, List years) { + public record TimeStratifiedInfos( + String label, + String description, + List columns, + Map totals, + List years) { } public record QuarterEntry(int quarter, Map values) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java index e59852310ac..6b959a69948 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java @@ -19,6 +19,7 @@ import com.bakdata.conquery.util.QueryUtils; import lombok.Getter; import lombok.ToString; +import org.jspecify.annotations.NonNull; /** * Represents the QueryPlan for corresponding to the {@link ArrayConceptQuery}. @@ -77,7 +78,7 @@ private static void initDateAggregator(DateAggregator validityDateAggregator, Li @Override public void init(QueryExecutionContext ctx, Entity entity) { validityDateAggregator.init(entity, ctx); - childPlans.forEach(child -> child.init(ctx,entity)); + childPlans.forEach(child -> child.init(ctx, entity)); } @Override @@ -94,7 +95,7 @@ public Optional execute(QueryExecutionContext ctx, Entit Object[] resultValues = new Object[getAggregatorSize()]; // Start with 1 for aggregator values if dateSet needs to be added to the result - final int resultOffset = generateDateAggregation ? 1 : 0; + final int resultOffset = generateDateAggregation ? 1 : 0; int resultInsertIdx = resultOffset; boolean containedInChildQueries = false; @@ -103,7 +104,9 @@ public Optional execute(QueryExecutionContext ctx, Entit if (result.isEmpty()) { // The sub result was empty. Generate the necessary gapped columns in the result line - final Object[] applied = ResultModifier.existAggValuesSetterFor(child.getAggregators(), OptionalInt.of(0)).apply(new Object[child.getAggregatorSize()]); + final Object[] applied = ResultModifier.existAggValuesSetterFor( + child.getAggregators(), + OptionalInt.of(0)).apply(new Object[child.getAggregatorSize()]); // applied[0] is the child-queries DateUnion, which we don't copy. int copyLength = applied.length - resultOffset; @@ -151,8 +154,8 @@ public boolean isOfInterest(Entity entity) { } @Override - public Optional> getValidityDateAggregator() { - if(!generateDateAggregation) { + public @NonNull Optional> getValidityDateAggregator() { + if (!generateDateAggregation) { return Optional.empty(); } @@ -193,7 +196,8 @@ private int nextIndex(int currentIdx, ConceptQueryPlan child) { */ int offset = child.getAggregatorSize() - (generateDateAggregation ? 1 : 0); if (offset < 0) { - throw new IllegalStateException("Result index offset must be positive, so the advancing pointer does not override results."); + throw new IllegalStateException( + "Result index offset must be positive, so the advancing pointer does not override results."); } return currentIdx + offset; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java index f23ed721306..db702c70e89 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java @@ -22,6 +22,7 @@ import lombok.Setter; import lombok.ToString; import lombok.extern.slf4j.Slf4j; +import org.jspecify.annotations.NonNull; @Getter @Setter @@ -29,10 +30,18 @@ @Slf4j public class ConceptQueryPlan implements QueryPlan { - private static final Meter INTERESTING_ENTITY = SharedMetricRegistries.getDefault().meter("queries.interest.entity.true"); - private static final Meter INTERESTING_BUCKET = SharedMetricRegistries.getDefault().meter("queries.interest.bucket.true"); - private static final Meter NOT_INTERESTING_ENTITY = SharedMetricRegistries.getDefault().meter("queries.interest.entity.false"); - private static final Meter NOT_INTERESTING_BUCKET = SharedMetricRegistries.getDefault().meter("queries.interest.bucket.false"); + private static final Meter INTERESTING_ENTITY = SharedMetricRegistries.getDefault() + .meter( + "queries.interest.entity.true"); + private static final Meter INTERESTING_BUCKET = SharedMetricRegistries.getDefault() + .meter( + "queries.interest.bucket.true"); + private static final Meter NOT_INTERESTING_ENTITY = SharedMetricRegistries.getDefault() + .meter( + "queries.interest.entity.false"); + private static final Meter NOT_INTERESTING_BUCKET = SharedMetricRegistries.getDefault() + .meter( + "queries.interest.bucket.false"); public static final int VALIDITY_DATE_POSITION = 0; @@ -166,14 +175,13 @@ private void reportEntityInterest(boolean interesting) { if (interesting) { INTERESTING_ENTITY.mark(); - } - else { + } else { NOT_INTERESTING_ENTITY.mark(); } } @Override - public Optional> getValidityDateAggregator() { + public @NonNull Optional> getValidityDateAggregator() { if (!isAggregateValidityDates()) { // The date aggregator was not added to the plan, so we don't collect a validity date return Optional.empty(); @@ -201,8 +209,7 @@ private void reportBucketInterest(boolean interesting) { if (interesting) { INTERESTING_BUCKET.mark(); - } - else { + } else { NOT_INTERESTING_BUCKET.mark(); } } @@ -211,4 +218,4 @@ public Set
collectRequiredTables() { return child.collectRequiredTables(); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregationAction.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregationAction.java index 98fea55885f..465b86bf151 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregationAction.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregationAction.java @@ -14,60 +14,60 @@ public enum DateAggregationAction { * Not propagate dates from lower levels to higher. */ BLOCK() { - @Override - public CDateSet aggregate(Set all) { - return null; - } - }, - /** - * Merge all collected dates from a lower level into a union. - */ - MERGE() { - @Override - public CDateSet aggregate(Set all) { + @Override + public CDateSet aggregate(Set all) { + return null; + } + }, + /** + * Merge all collected dates from a lower level into a union. + */ + MERGE() { + @Override + public CDateSet aggregate(Set all) { CDateSet combined = CDateSet.createEmpty(); - all.forEach(combined::addAll); - return combined; - } - }, - /** - * Intersect all collected date sets from lower level with each other. - */ - INTERSECT() { - @Override - public CDateSet aggregate(Set all) { - if (all.size() < 1) { + all.forEach(combined::addAll); + return combined; + } + }, + /** + * Intersect all collected date sets from lower level with each other. + */ + INTERSECT() { + @Override + public CDateSet aggregate(Set all) { + if (all.size() < 1) { return CDateSet.createEmpty(); - } + } - Iterator it = all.iterator(); - CDateSet intersection = it.next(); + Iterator it = all.iterator(); + CDateSet intersection = it.next(); - if (all.size() == 1) { - return intersection; - } - // Use the first range as mask and subtract all other ranges from it + if (all.size() == 1) { + return intersection; + } + // Use the first range as mask and subtract all other ranges from it - // Intersect - while (it.hasNext()) { - final CDateSet next = it.next(); - intersection.retainAll(next); - } - return intersection; - } - }, - /** - * Build the negative of the the union of all collected dates - */ - NEGATE() { - @Override - public CDateSet aggregate(Set all) { - CDateSet negative = CDateSet.createFull(); - all.forEach(negative::removeAll); - return negative; - } - }; + // Intersect + while (it.hasNext()) { + final CDateSet next = it.next(); + intersection.retainAll(next); + } + return intersection; + } + }, + /** + * Build the negative of the the union of all collected dates + */ + NEGATE() { + @Override + public CDateSet aggregate(Set all) { + CDateSet negative = CDateSet.createFull(); + all.forEach(negative::removeAll); + return negative; + } + }; - public abstract CDateSet aggregate(Set all); + public abstract CDateSet aggregate(Set all); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java index 46dac03a6a6..2d3fcc6ffdf 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java @@ -21,52 +21,53 @@ @ToString(of = "action") public class DateAggregator extends Aggregator { - private final DateAggregationAction action; + private final DateAggregationAction action; - private final Set> children = new HashSet<>(); + private final Set> children = new HashSet<>(); - /** - * Register {@Aggregator}s from lower levels for the final result generation. - */ - public void register(Aggregator child) { + /** + * Register {@Aggregator}s from lower levels for the final result generation. + */ + public void register(Aggregator child) { children.add(child); - } + } - /** - * Register {@Aggregator}s from lower levels for the final result generation. - */ - public void registerAll(Collection> children) { - this.children.addAll(children); - } + /** + * Register {@Aggregator}s from lower levels for the final result generation. + */ + public void registerAll(Collection> children) { + this.children.addAll(children); + } @Override public void init(Entity entity, QueryExecutionContext context) { - //TODO don't think this is needed? + //TODO don't think this is needed? children.forEach(child -> child.init(entity, context)); } @Override - public void consumeEvent(Bucket bucket, int event) { - throw new UnsupportedOperationException("This Aggregator uses the result of its siblings and does not accept events"); - } + public void consumeEvent(Bucket bucket, int event) { + throw new UnsupportedOperationException( + "This Aggregator uses the result of its siblings and does not accept events"); + } - @Override - public CDateSet createAggregationResult() { - final Set all = new HashSet<>(); - children.forEach(s -> { - CDateSet result = s.createAggregationResult(); - if(result != null) { - all.add(result); - } - }); + @Override + public CDateSet createAggregationResult() { + final Set all = new HashSet<>(); + children.forEach(s -> { + CDateSet result = s.createAggregationResult(); + if (result != null) { + all.add(result); + } + }); - // Repackage to get the results sorted. Might need some optimization. - return action.aggregate(all); - } + // Repackage to get the results sorted. Might need some optimization. + return action.aggregate(all); + } public boolean hasChildren() { - return !children.isEmpty(); - } + return !children.isEmpty(); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPChainNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPChainNode.java index bfaa6c05af1..4ccc37496ee 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPChainNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPChainNode.java @@ -28,8 +28,7 @@ public QPChainNode(QPNode child) { public void setChild(QPNode child) { if (child == null) { this.child = new Leaf(); - } - else { + } else { this.child = child; } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPNode.java index 42c7ba6a433..3944e392d55 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPNode.java @@ -14,7 +14,8 @@ import lombok.Getter; import lombok.Setter; -@Getter(AccessLevel.PROTECTED) @Setter(AccessLevel.PROTECTED) +@Getter(AccessLevel.PROTECTED) +@Setter(AccessLevel.PROTECTED) public abstract class QPNode extends EventIterating { protected QueryExecutionContext context; protected Entity entity; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPParentNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPParentNode.java index 529e86860eb..b84a3aa7d70 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPParentNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPParentNode.java @@ -44,14 +44,11 @@ public QPParentNode(List children, DateAggregationAction action) { } this.children = children; childMap = children.stream() - .flatMap( - c -> c.collectRequiredTables() - .stream() - .map(t -> Pair.of(t, c)) - ) - .collect(ImmutableListMultimap - .toImmutableListMultimap(Pair::getLeft, Pair::getRight) - ); + .flatMap( + c -> c.collectRequiredTables().stream().map(t -> Pair.of(t, c)) + ) + .collect(ImmutableListMultimap.toImmutableListMultimap(Pair::getLeft, Pair::getRight) + ); // Save action for debugging this.action = action; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QueryPlan.java index f0480a9ce8a..73ca3e14f6d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QueryPlan.java @@ -19,4 +19,4 @@ public interface QueryPlan { @NotNull Optional> getValidityDateAggregator(); -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java index 31b84315002..6641a62b687 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java @@ -30,6 +30,7 @@ import lombok.Setter; import lombok.ToString; import org.apache.commons.lang3.ArrayUtils; +import org.jspecify.annotations.NonNull; /** * This class is able to execute a typical ConceptQueryPlan, but will create @@ -113,7 +114,10 @@ private void executeQueriesWithSecondaryId(QueryExecutionContext ctx, Entity ent nextTable(ctxWithPhase, currentTable); - final Set tableBuckets = ctx.getBucketManager().getEntityBucketsForTable(entity, currentTable.getId()); + final Set tableBuckets = ctx.getBucketManager() + .getEntityBucketsForTable( + entity, + currentTable.getId()); for (BucketId bucketId : tableBuckets) { Bucket bucket = bucketId.resolve(); @@ -153,8 +157,7 @@ private void executeQueriesWithSecondaryId(QueryExecutionContext ctx, Entity ent if (consumed) { childPerKey.put(key, plan); - } - else { + } else { discardSubPlan(plan); } @@ -166,7 +169,10 @@ private void executeQueriesWithoutSecondaryId(QueryExecutionContext ctx, Entity nextTable(ctx, currentTable); - final Set tableBuckets = ctx.getBucketManager().getEntityBucketsForTable(entity, currentTable.getId()); + final Set tableBuckets = ctx.getBucketManager() + .getEntityBucketsForTable( + entity, + currentTable.getId()); for (BucketId bucketId : tableBuckets) { Bucket bucket = bucketId.resolve(); @@ -213,7 +219,9 @@ private Optional createResult(Entity entity) { private void nextTable(QueryExecutionContext ctx, Table currentTable) { queryPlan.nextTable(ctx, currentTable); for (ConceptQueryPlan c : childPerKey.values()) { - final QueryExecutionContext context = QueryUtils.determineDateAggregatorForContext(ctx, c::getValidityDateAggregator); + final QueryExecutionContext context = QueryUtils.determineDateAggregatorForContext( + ctx, + c::getValidityDateAggregator); c.nextTable(context, currentTable); } } @@ -244,7 +252,9 @@ private ConceptQueryPlan createChild(QueryExecutionContext currentContext, Bucke plan = query.createQueryPlan(queryPlanContext.withSelectedSecondaryId(secondaryId)); } - final QueryExecutionContext context = QueryUtils.determineDateAggregatorForContext(currentContext, plan::getValidityDateAggregator); + final QueryExecutionContext context = QueryUtils.determineDateAggregatorForContext( + currentContext, + plan::getValidityDateAggregator); plan.init(context, queryPlan.getEntity()); plan.nextTable(context, currentBucket.getTable().resolve()); @@ -264,7 +274,7 @@ public boolean isOfInterest(Entity entity) { } @Override - public Optional> getValidityDateAggregator() { + public @NonNull Optional> getValidityDateAggregator() { if (!queryPlan.isAggregateValidityDates()) { return Optional.empty(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/TableExportQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/TableExportQueryPlan.java index e431c7a1e63..d59dddb6b78 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/TableExportQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/TableExportQueryPlan.java @@ -78,7 +78,10 @@ public Optional execute(QueryExecutionContext ctx, Entity final CQTable cqTable = entry.getKey(); final ValidityDate validityDate = cqTable.findValidityDate(); final QPNode query = entry.getValue(); - final Map cblocks = ctx.getBucketManager().getEntityCBlocksForConnector(entity, cqTable.getConnector()); + final Map cblocks = ctx.getBucketManager() + .getEntityCBlocksForConnector( + entity, + cqTable.getConnector()); final Connector connector = cqTable.getConnector().resolve(); for (BucketId bucketId : ctx.getEntityBucketsForTable(entity, connector.resolveTableId())) { @@ -94,8 +97,7 @@ public Optional execute(QueryExecutionContext ctx, Entity for (int event = start; event < end; event++) { - if (validityDate != null - && !bucket.eventIsContainedIn(event, validityDate, dateRange)) { + if (validityDate != null && !bucket.eventIsContainedIn(event, validityDate, dateRange)) { continue; } @@ -157,13 +159,19 @@ private boolean isRowIncluded(QPNode query, Bucket bucket, Entity entity, int ev return query.isContained(); } - private Object[] collectRow(int totalColumns, CQTable exportDescription, Bucket bucket, int event, ValidityDate validityDate, CBlock cblock) { + private Object[] collectRow( + int totalColumns, + CQTable exportDescription, + Bucket bucket, + int event, + ValidityDate validityDate, + CBlock cblock) { final Object[] entry = new Object[totalColumns]; final CDateRange date; - if(validityDate != null && (date = validityDate.getValidityDate(event, bucket)) != null) { + if (validityDate != null && (date = validityDate.getValidityDate(event, bucket)) != null) { entry[0] = List.of(date); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/Aggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/Aggregator.java index 20c1bea43c6..08420875d53 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/Aggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/Aggregator.java @@ -2,8 +2,6 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.query.queryplan.EventIterating; -import com.bakdata.conquery.models.types.ResultType; -import com.fasterxml.jackson.annotation.JsonIgnore; /** * An aggregator iterates over events, computing a value alongside. Values are fed through {@link Aggregator#consumeEvent(Bucket, int)}, and the result can be queried at {@link Aggregator#createAggregationResult()}. @@ -33,7 +31,7 @@ public boolean acceptEvent(Bucket bucket, int event) { return true; } - public abstract void consumeEvent(Bucket bucket, int event) ; + public abstract void consumeEvent(Bucket bucket, int event); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/DistinctValuesWrapperAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/DistinctValuesWrapperAggregator.java index 9b691755776..2dceb14eaa0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/DistinctValuesWrapperAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/DistinctValuesWrapperAggregator.java @@ -39,10 +39,7 @@ public VALUE createAggregationResult() { @Override public List getRequiredColumns() { - return ImmutableList.builder() - .addAll(aggregator.getRequiredColumns()) - .addAll(getColumns()) - .build(); + return ImmutableList.builder().addAll(aggregator.getRequiredColumns()).addAll(getColumns()).build(); } @Override @@ -81,8 +78,7 @@ public void consumeEvent(Bucket bucket, int event) { if (bucket.has(event, column)) { anyPresent = true; incoming.add(bucket.createScriptValue(event, column)); - } - else { + } else { incoming.add(null); } } @@ -92,4 +88,4 @@ public void consumeEvent(Bucket bucket, int event) { } } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountAggregator.java index 56f3aa0b306..09bf0af553d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountAggregator.java @@ -2,7 +2,6 @@ import java.util.List; import java.util.Set; - import javax.annotation.Nullable; import com.bakdata.conquery.models.datasets.Column; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateDistanceAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateDistanceAggregator.java index e44518c1a8b..5e982932cd1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateDistanceAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateDistanceAggregator.java @@ -38,8 +38,7 @@ public void init(Entity entity, QueryExecutionContext context) { public void nextTable(QueryExecutionContext ctx, Table currentTable) { if (CDate.isPositiveInfinity(ctx.getDateRestriction().getMaxValue()) || ctx.getDateRestriction().isEmpty()) { reference = CDate.toLocalDate(ctx.getToday()); - } - else { + } else { reference = CDate.toLocalDate(ctx.getDateRestriction().getMaxValue()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDateUnionAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDateUnionAggregator.java index 681c230fef7..9ea72279d83 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDateUnionAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDateUnionAggregator.java @@ -72,14 +72,14 @@ public CDateSet createAggregationResult() { @Override public void consumeEvent(Bucket bucket, int event) { - if(validityDateColumn == null) { + if (validityDateColumn == null) { set.addAll(dateRestriction); return; } final CDateRange dateRange = validityDateColumn.getValidityDate(event, bucket); - if (dateRange == null){ + if (dateRange == null) { return; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDurationSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDurationSumAggregator.java index c1a3fd46fdf..42de5b77c78 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDurationSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDurationSumAggregator.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.models.query.queryplan.aggregators.specific; import java.util.Optional; - import javax.annotation.CheckForNull; import com.bakdata.conquery.models.common.CDateSet; @@ -49,7 +48,7 @@ public void consumeEvent(Bucket bucket, int event) { final CDateRange value = validityDateColumn.getValidityDate(event, bucket); - if (value == null){ + if (value == null) { return; } @@ -60,11 +59,10 @@ public void consumeEvent(Bucket bucket, int event) { @Override public Long createAggregationResult() { - queryDateAggregator - .map(Aggregator::createAggregationResult) - .ifPresent( - set::retainAll - ); + queryDateAggregator.map(Aggregator::createAggregationResult) + .ifPresent( + set::retainAll + ); return set.countDays(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ExistsAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ExistsAggregator.java index a146f648fb4..be89e2fdb39 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ExistsAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ExistsAggregator.java @@ -25,13 +25,14 @@ public class ExistsAggregator extends Aggregator { private QPNode reference; @Override - public void consumeEvent(Bucket bucket, int event) { } + public void consumeEvent(Bucket bucket, int event) { + } @Override public Boolean createAggregationResult() { return reference.isContained(); } - + @Override public void collectRequiredTables(Set
requiredTables) { requiredTables.addAll(this.requiredTables); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuarterAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuarterAggregator.java index 8058835bb1f..b6544e6c32d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuarterAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuarterAggregator.java @@ -55,7 +55,7 @@ public void nextTable(QueryExecutionContext ctx, Table currentTable) { public void consumeEvent(Bucket bucket, int event) { final CDateRange dateRange = validityDate.getValidityDate(event, bucket); - if (dateRange == null){ + if (dateRange == null) { return; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuartersInYearAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuartersInYearAggregator.java index b4a3292c1f6..e40b62df6de 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuartersInYearAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuartersInYearAggregator.java @@ -47,15 +47,14 @@ public void consumeEvent(Bucket bucket, int event) { if (months == null) { months = EnumSet.of(QuarterUtils.getFirstMonthOfQuarter(quarter)); quartersInYear.put(date.getYear(), months); - } - else { + } else { months.add(QuarterUtils.getFirstMonthOfQuarter(quarter)); } } @Override public Long createAggregationResult() { - if(quartersInYear.isEmpty()) { + if (quartersInYear.isEmpty()) { return null; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SpecialDateUnion.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SpecialDateUnion.java index 0f99a92aadd..a488bfc50a8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SpecialDateUnion.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SpecialDateUnion.java @@ -44,7 +44,7 @@ public void consumeEvent(Bucket bucket, int event) { final CDateRange dateRange = validityDate.getValidityDate(event, bucket); - if (dateRange == null){ + if (dateRange == null) { set.addAll(dateRestriction); return; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/TwoColumnDurationSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/TwoColumnDurationSumAggregator.java index a36bf62cdc6..e8b9243d3ce 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/TwoColumnDurationSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/TwoColumnDurationSumAggregator.java @@ -10,7 +10,6 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; -import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.ColumnAggregator; import lombok.Data; import lombok.ToString; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/DecimalDiffSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/DecimalDiffSumAggregator.java index ce76bd96e9d..7cb54727d41 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/DecimalDiffSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/DecimalDiffSumAggregator.java @@ -56,9 +56,13 @@ public void consumeEvent(Bucket bucket, int event) { hit = true; - BigDecimal addend = bucket.has(event, getAddendColumn()) ? bucket.getDecimal(event, getAddendColumn()) : BigDecimal.ZERO; + BigDecimal addend = bucket.has(event, getAddendColumn()) ? bucket.getDecimal( + event, + getAddendColumn()) : BigDecimal.ZERO; - BigDecimal subtrahend = bucket.has(event, getSubtrahendColumn()) ? bucket.getDecimal(event, getSubtrahendColumn()) : BigDecimal.ZERO; + BigDecimal subtrahend = bucket.has(event, getSubtrahendColumn()) ? bucket.getDecimal( + event, + getSubtrahendColumn()) : BigDecimal.ZERO; sum = sum.add(addend.subtract(subtrahend)); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/IntegerDiffSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/IntegerDiffSumAggregator.java index 9c24a6a8632..46ddae176bd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/IntegerDiffSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/IntegerDiffSumAggregator.java @@ -46,6 +46,7 @@ public List getRequiredColumns() { return out; } + @Override public void consumeEvent(Bucket bucket, int event) { @@ -56,7 +57,9 @@ public void consumeEvent(Bucket bucket, int event) { hit = true; long addend = bucket.has(event, getAddendColumn()) ? bucket.getInteger(event, getAddendColumn()) : 0; - long subtrahend = bucket.has(event, getSubtrahendColumn()) ? bucket.getInteger(event, getSubtrahendColumn()) : 0; + long subtrahend = bucket.has(event, getSubtrahendColumn()) ? bucket.getInteger( + event, + getSubtrahendColumn()) : 0; sum = sum + addend - subtrahend; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/MoneyDiffSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/MoneyDiffSumAggregator.java index 7360e7ad0c6..4e17fab1008 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/MoneyDiffSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/MoneyDiffSumAggregator.java @@ -56,9 +56,13 @@ public void consumeEvent(Bucket bucket, int event) { hit = true; - final BigDecimal addend = bucket.has(event, getAddendColumn()) ? bucket.getMoney(event, getAddendColumn()) : BigDecimal.ZERO; + final BigDecimal addend = bucket.has(event, getAddendColumn()) ? bucket.getMoney( + event, + getAddendColumn()) : BigDecimal.ZERO; - final BigDecimal subtrahend = bucket.has(event, getSubtrahendColumn()) ? bucket.getMoney(event, getSubtrahendColumn()) : BigDecimal.ZERO; + final BigDecimal subtrahend = bucket.has(event, getSubtrahendColumn()) ? bucket.getMoney( + event, + getSubtrahendColumn()) : BigDecimal.ZERO; sum = sum.add(addend).subtract(subtrahend); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/RealDiffSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/RealDiffSumAggregator.java index 9376c36f055..0caeefe4dd3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/RealDiffSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/RealDiffSumAggregator.java @@ -56,13 +56,9 @@ public void consumeEvent(Bucket bucket, int event) { hit = true; - double addend = bucket.has(event, getAddendColumn()) - ? bucket.getReal(event, getAddendColumn()) - : 0; + double addend = bucket.has(event, getAddendColumn()) ? bucket.getReal(event, getAddendColumn()) : 0; - double subtrahend = bucket.has(event, getSubtrahendColumn()) - ? bucket.getReal(event, getSubtrahendColumn()) - : 0; + double subtrahend = bucket.has(event, getSubtrahendColumn()) ? bucket.getReal(event, getSubtrahendColumn()) : 0; sum = sum + addend - subtrahend; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/package-info.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/package-info.java index f1885fbb9c0..2492d874628 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/package-info.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/package-info.java @@ -1,4 +1,4 @@ /** * Implementations of Aggregators. */ -package com.bakdata.conquery.models.query.queryplan.aggregators.specific; \ No newline at end of file +package com.bakdata.conquery.models.query.queryplan.aggregators.specific; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/ConceptElementsAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/ConceptElementsAggregator.java index 6d018c772a2..2470ae8a2c3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/ConceptElementsAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/ConceptElementsAggregator.java @@ -35,9 +35,11 @@ public class ConceptElementsAggregator extends Aggregator> { public ConceptElementsAggregator(TreeConcept concept) { super(); - tableConnectors = concept.getConnectors().stream() - .filter(conn -> conn.getColumn() != null) - .collect(Collectors.toMap(Connector::getResolvedTable, Functions.identity())); + tableConnectors = concept.getConnectors() + .stream() + .filter(conn -> conn.getColumn() != null) + .collect( + Collectors.toMap(Connector::getResolvedTable, Functions.identity())); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/ConceptValuesAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/ConceptValuesAggregator.java index 773e8138619..a0b1d763962 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/ConceptValuesAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/ConceptValuesAggregator.java @@ -31,9 +31,11 @@ public class ConceptValuesAggregator extends Aggregator> { public ConceptValuesAggregator(TreeConcept concept) { super(); this.concept = concept; - tableConnectors = concept.getConnectors().stream() - .filter(conn -> conn.getColumn() != null) - .collect(Collectors.toMap(Connector::getResolvedTable, Functions.identity())); + tableConnectors = concept.getConnectors() + .stream() + .filter(conn -> conn.getColumn() != null) + .collect( + Collectors.toMap(Connector::getResolvedTable, Functions.identity())); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/FirstValueAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/FirstValueAggregator.java index 61980f6f052..989f61584c5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/FirstValueAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/FirstValueAggregator.java @@ -60,9 +60,10 @@ public void consumeEvent(Bucket bucket, int event) { if (selectedBucket == null) { selectedBucket = bucket; selectedEvent = event; - } - else { - log.trace("There is more than one value for the {}. Choosing the very first one encountered", getClass().getSimpleName()); + } else { + log.trace( + "There is more than one value for the {}. Choosing the very first one encountered", + getClass().getSimpleName()); } return; } @@ -79,9 +80,10 @@ public void consumeEvent(Bucket bucket, int event) { date = next; selectedEvent = event; selectedBucket = bucket; - } - else if (next == date) { - log.trace("There is more than one value for the {}. Choosing the very first one encountered", getClass().getSimpleName()); + } else if (next == date) { + log.trace( + "There is more than one value for the {}. Choosing the very first one encountered", + getClass().getSimpleName()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/LastValueAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/LastValueAggregator.java index 8dc70461266..3682a900487 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/LastValueAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/LastValueAggregator.java @@ -52,21 +52,23 @@ public void consumeEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { return; } - + if (validityDateColumn == null) { // If there is no validity date, take the first possible value - if(selectedBucket == null) { + if (selectedBucket == null) { selectedBucket = bucket; selectedEvent = event; } else { - log.trace("There is more than one value for the {}. Choosing the very first one encountered", this.getClass().getSimpleName()); + log.trace( + "There is more than one value for the {}. Choosing the very first one encountered", + this.getClass().getSimpleName()); } - return; + return; } final CDateRange dateRange = validityDateColumn.getValidityDate(event, bucket); - if (dateRange == null){ + if (dateRange == null) { return; } @@ -76,9 +78,10 @@ public void consumeEvent(Bucket bucket, int event) { date = next; selectedEvent = event; selectedBucket = bucket; - } - else if (next == date) { - log.trace("There is more than one value for the {}. Choosing the very first one encountered", this.getClass().getSimpleName()); + } else if (next == date) { + log.trace( + "There is more than one value for the {}. Choosing the very first one encountered", + this.getClass().getSimpleName()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/FilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/FilterNode.java index ce7ca5f059a..5f99aa9e96f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/FilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/FilterNode.java @@ -9,9 +9,11 @@ @AllArgsConstructor @ToString -public abstract sealed class FilterNode extends EventIterating permits EventFilterNode, AggregationFilterNode { +public abstract sealed class FilterNode extends EventIterating permits EventFilterNode, + AggregationFilterNode { - @Setter(AccessLevel.PROTECTED) @Getter + @Setter(AccessLevel.PROTECTED) + @Getter protected FILTER_VALUE filterValue; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/package-info.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/package-info.java index 82e71979203..ded7081b471 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/package-info.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/package-info.java @@ -1,4 +1,4 @@ /** * Package containing classes that operate on {@link com.bakdata.conquery.models.query.entity.Entity} and {@link com.bakdata.conquery.models.events.Block} objects: Specifically Aggregation and Filtering of values. */ -package com.bakdata.conquery.models.query.queryplan; \ No newline at end of file +package com.bakdata.conquery.models.query.queryplan; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ConceptNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ConceptNode.java index 5ceaaf9bd5e..d1c17df34a8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ConceptNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ConceptNode.java @@ -34,15 +34,24 @@ public class ConceptNode extends QPChainNode { private Map preCurrentRow; private CBlock currentRow; - public ConceptNode(QPNode child, List> concepts, CQTable table, SecondaryIdDescription selectedSecondaryId) { + public ConceptNode( + QPNode child, + List> concepts, + CQTable table, + SecondaryIdDescription selectedSecondaryId) { this(child, concepts, calculateBitMask(concepts), table, selectedSecondaryId); } // For cloning - private ConceptNode(QPNode child, List> concepts, long requiredBits, CQTable table, SecondaryIdDescription selectedSecondaryId) { + private ConceptNode( + QPNode child, + List> concepts, + long requiredBits, + CQTable table, + SecondaryIdDescription selectedSecondaryId) { super(child); this.concepts = concepts; - this.requiredBits = requiredBits; + this.requiredBits = requiredBits; this.table = table; this.selectedSecondaryId = selectedSecondaryId; @@ -64,15 +73,16 @@ public static long calculateBitMask(Collection> concepts) { @Override public void init(Entity entity, QueryExecutionContext context) { super.init(entity, context); - preCurrentRow = context.getBucketManager().getEntityCBlocksForConnector(getEntity(),table.getConnector()); + preCurrentRow = context.getBucketManager().getEntityCBlocksForConnector(getEntity(), table.getConnector()); } @Override public void nextTable(QueryExecutionContext ctx, Table currentTable) { Connector connector = table.getConnector().resolve(); - tableActive = connector.resolveTableId().equals(currentTable.getId()) - && ctx.getActiveSecondaryId() == selectedSecondaryId; - if(tableActive) { + tableActive = connector.resolveTableId() + .equals( + currentTable.getId()) && ctx.getActiveSecondaryId() == selectedSecondaryId; + if (tableActive) { super.nextTable(ctx.withConnector(connector), currentTable); } } @@ -88,8 +98,10 @@ public void nextBlock(Bucket bucket) { @Override public boolean isOfInterest(Entity entity) { - return context.getBucketManager().hasEntityCBlocksForConnector(entity, table.getConnector()) - && getChild().isOfInterest(entity); + return context.getBucketManager() + .hasEntityCBlocksForConnector( + entity, + table.getConnector()) && getChild().isOfInterest(entity); } @Override @@ -98,13 +110,13 @@ public boolean isOfInterest(Bucket bucket) { return false; } - if(!bucket.containsEntity(getEntity().getId())){ + if (!bucket.containsEntity(getEntity().getId())) { return false; } final CBlock cBlock = Objects.requireNonNull(preCurrentRow.get(bucket.getId()).resolve()); - if(cBlock.isConceptIncluded(entity.getId(), requiredBits)) { + if (cBlock.isConceptIncluded(entity.getId(), requiredBits)) { return super.isOfInterest(bucket); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java index e5455c14870..ca9631f70ba 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.query.queryplan.specific; +import jakarta.validation.constraints.NotEmpty; import java.util.Collection; import java.util.Collections; import java.util.Map; @@ -13,8 +14,6 @@ import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.ConstantValueAggregator; - -import jakarta.validation.constraints.NotEmpty; import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.ToString; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/FiltersNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/FiltersNode.java index 2153b5c5a00..b94d5bb227d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/FiltersNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/FiltersNode.java @@ -34,7 +34,10 @@ public class FiltersNode extends QPNode { private final EventDateUnionAggregator eventDateAggregator; private boolean hit = false; - public static FiltersNode create(List> filters, List> aggregators, EventDateUnionAggregator eventDateAggregator) { + public static FiltersNode create( + List> filters, + List> aggregators, + EventDateUnionAggregator eventDateAggregator) { if (filters.isEmpty() && aggregators.isEmpty()) { throw new IllegalStateException("Unable to create FilterNode without filters or aggregators."); } @@ -51,11 +54,11 @@ public static FiltersNode create(List> filters, List> getDateAggregators() { public boolean isOfInterest(Bucket bucket) { return bucket.containsEntity(entity.getId()); } - + @Override public boolean isOfInterest(Entity entity) { return true; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/NegatingNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/NegatingNode.java index c9989ab2e40..7604e1d9485 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/NegatingNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/NegatingNode.java @@ -29,7 +29,7 @@ private NegatingNode(@NonNull QPNode child, @NonNull DateAggregator dateAggregat super(child); this.dateAggregator = dateAggregator; } - + @Override public boolean acceptEvent(Bucket bucket, int event) { return getChild().acceptEvent(bucket, event); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/OrNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/OrNode.java index 2a403d3c234..a33652b6d34 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/OrNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/OrNode.java @@ -16,7 +16,7 @@ public class OrNode extends QPParentNode { public OrNode(List children, DateAggregationAction action) { super(children, action); } - + private OrNode(List children, ListMultimap childMap, DateAggregator dateAggregator) { super(children, childMap, dateAggregator); } @@ -25,12 +25,12 @@ private OrNode(List children, ListMultimap childMap, Date @Override public boolean isContained() { boolean currently = false; - for(QPNode agg:getChildren()) { + for (QPNode agg : getChildren()) { currently |= agg.isContained(); } return currently; } - + public static QPNode of(Collection children, DateAggregationAction dateAggregationAction) { return switch (children.size()) { case 0 -> new Leaf(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ValidityDateNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ValidityDateNode.java index 0a8601b6a82..84bba5c2126 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ValidityDateNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ValidityDateNode.java @@ -34,7 +34,7 @@ public boolean acceptEvent(Bucket bucket, int event) { //no dateRestriction or event is in date restriction final boolean contained = bucket.eventIsContainedIn(event, validityDate, context.getDateRestriction()); - if (!contained){ + if (!contained) { return false; } @@ -46,7 +46,10 @@ public void nextTable(QueryExecutionContext ctx, Table currentTable) { super.nextTable(ctx.withValidityDateColumn(validityDate), currentTable); restriction = ctx.getDateRestriction(); - preCurrentRow = ctx.getBucketManager().getEntityCBlocksForConnector(getEntity(), context.getConnector().getId()); + preCurrentRow = ctx.getBucketManager() + .getEntityCBlocksForConnector( + getEntity(), + context.getConnector().getId()); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Yes.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Yes.java index ec8e65c15b7..2ea39453236 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Yes.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Yes.java @@ -49,7 +49,7 @@ public Collection> getDateAggregators() { public boolean isOfInterest(Bucket bucket) { return true; } - + @Override public boolean isOfInterest(Entity entity) { return true; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalQueryNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalQueryNode.java index 0c5987ca46f..701c7ee71d5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalQueryNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalQueryNode.java @@ -165,21 +165,24 @@ public boolean evaluateTemporalQuery(QueryExecutionContext ctx, Entity entity) { if (compareContained[inner]) { compareDates[inner] = innerCompareQueryPlan.getDateAggregator().createAggregationResult(); - compareAggregationResults[inner] = innerCompareQueryPlan.getAggregators().stream() - .skip(1) // Skip the result-date - .map(Aggregator::createAggregationResult).toArray(); + compareAggregationResults[inner] = innerCompareQueryPlan.getAggregators() + .stream() + .skip(1) // Skip the result-date + .map(Aggregator::createAggregationResult) + .toArray(); } } boolean satisfies = compareSelector.satisfies(compareContained); - log.trace("{}:{} => indexPeriod={}, comparePeriods={}, compareContained={} => {}", - getEntity(), - compareSelector, - indexPeriod, - comparePeriods, - compareContained, - satisfies + log.trace( + "{}:{} => indexPeriod={}, comparePeriods={}, compareContained={} => {}", + getEntity(), + compareSelector, + indexPeriod, + comparePeriods, + compareContained, + satisfies ); results[current] = satisfies; @@ -195,7 +198,13 @@ public boolean evaluateTemporalQuery(QueryExecutionContext ctx, Entity entity) { boolean satisfies = indexSelector.satisfies(results); - log.trace("{}:{} => indexPeriods={}, results={} => {}", getEntity(), indexSelector, indexPeriods, results, satisfies); + log.trace( + "{}:{} => indexPeriods={}, results={} => {}", + getEntity(), + indexSelector, + indexPeriods, + results, + satisfies); return satisfies; @@ -206,9 +215,12 @@ public boolean evaluateTemporalQuery(QueryExecutionContext ctx, Entity entity) { * * @return if the entity is contained or not. */ - private static boolean evaluateWithRestriction(Entity entity, CDateRange partition, ConceptQueryPlan cqp, QueryExecutionContext ctx) { - ctx = ctx.withDateRestriction(CDateSet.create(partition)) - .withQueryDateAggregator(Optional.empty()); + private static boolean evaluateWithRestriction( + Entity entity, + CDateRange partition, + ConceptQueryPlan cqp, + QueryExecutionContext ctx) { + ctx = ctx.withDateRestriction(CDateSet.create(partition)).withQueryDateAggregator(Optional.empty()); cqp.init(ctx, entity); cqp.execute(ctx, entity); @@ -220,7 +232,10 @@ private static boolean evaluateWithRestriction(Entity entity, CDateRange partiti * Collect aggregation results of satisfied innerQueries. */ @SuppressWarnings("unchecked") - private void addAggregationResults(boolean[] compareContained, CDateSet[] compareDates, Object[][] compareAggregationResults) { + private void addAggregationResults( + boolean[] compareContained, + CDateSet[] compareDates, + Object[][] compareAggregationResults) { assert compareContained.length == compareDates.length; assert compareContained.length == compareAggregationResults.length; @@ -232,9 +247,7 @@ private void addAggregationResults(boolean[] compareContained, CDateSet[] compar compareDateResult.addAll(compareDates[index]); for (int aggIndex = 0; aggIndex < aggregationResults.size(); aggIndex++) { - aggregationResults.get(aggIndex) - .getValue() - .add(compareAggregationResults[index][aggIndex]); + aggregationResults.get(aggIndex).getValue().add(compareAggregationResults[index][aggIndex]); } } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalRelationMode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalRelationMode.java index 7eebc145fa4..98cb0ac651d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalRelationMode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalRelationMode.java @@ -17,7 +17,8 @@ */ @JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "type") @CPSBase -sealed public interface TemporalRelationMode permits TemporalRelationMode.After, TemporalRelationMode.Before, TemporalRelationMode.While { +sealed public interface TemporalRelationMode permits TemporalRelationMode.After, TemporalRelationMode.Before, + TemporalRelationMode.While { /** * Take result-dates and convert it to date-restrictions to assert the temporal relation. diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalSelector.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalSelector.java index 347e1cc7556..7c5d197b0db 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalSelector.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalSelector.java @@ -18,7 +18,8 @@ public CDateRange[] sample(CDateSet result) { public boolean satisfies(boolean[] results) { return results.length > 0 && BooleanUtils.or(results); } - }, ALL { + }, + ALL { @Override public CDateRange[] sample(CDateSet result) { return result.asRanges().toArray(CDateRange[]::new); @@ -28,7 +29,8 @@ public CDateRange[] sample(CDateSet result) { public boolean satisfies(boolean[] results) { return results.length > 0 && BooleanUtils.and(results); } - }, EARLIEST { + }, + EARLIEST { @Override public CDateRange[] sample(CDateSet result) { return new CDateRange[]{result.asRanges().iterator().next()}; @@ -39,7 +41,8 @@ public boolean satisfies(boolean[] results) { return results.length == 1 && results[0]; } - }, LATEST { + }, + LATEST { @Override public CDateRange[] sample(CDateSet result) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java index a55cae4aadf..921d3364f4e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ColumnResultInfo.java @@ -46,7 +46,7 @@ public String defaultColumnName(PrintSettings printSettings) { @Override public Printer createPrinter(PrinterFactory printerFactory, PrintSettings printSettings) { - if(concept != null){ + if (concept != null) { return new ConceptIdPrinter(concept, printSettings); } return printerFactory.printerFor(type, printSettings); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java index e48f1295408..c5b8e0ba019 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java @@ -41,10 +41,11 @@ public final void addSemantics(SemanticType... incoming) { public final ColumnDescriptor asColumnDescriptor(UniqueNamer collector, PrintSettings printSettings) { return new ColumnDescriptor( - collector.getUniqueName(this, printSettings), - defaultColumnName(printSettings), getDescription(), - getType().typeInfo(), - getSemantics() + collector.getUniqueName(this, printSettings), + defaultColumnName(printSettings), + getDescription(), + getType().typeInfo(), + getSemantics() ); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/UniqueNamer.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/UniqueNamer.java index 12218e1edfb..afabfa6aca8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/UniqueNamer.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/UniqueNamer.java @@ -34,8 +34,9 @@ public class UniqueNamer { @NonNull @JsonIgnore public final String getUniqueName(ResultInfo info, PrintSettings printSettings) { - @NonNull - String label = Objects.requireNonNullElse(info.userColumnName(printSettings), info.defaultColumnName(printSettings)); + @NonNull String label = Objects.requireNonNullElse( + info.userColumnName(printSettings), + info.defaultColumnName(printSettings)); // lookup if prefix is needed and computed it if necessary String uniqueName = label; synchronized (ocurrenceCounter) { @@ -51,4 +52,4 @@ public final String getUniqueName(ResultInfo info, PrintSettings printSettings) } return uniqueName; } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java index fe5684e69d7..1d32ae49579 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ArrowResultPrinters.java @@ -24,7 +24,7 @@ public Printer getMoneyPrinter(PrintSettings printSettings) { private record MoneyPrinter() implements Printer { @Override public Object apply(@NotNull Number value) { - if (value instanceof BigDecimal bigDecimal){ + if (value instanceof BigDecimal bigDecimal) { if (bigDecimal.scale() > 2) { log.warn("Rounding money value '{}' as scale is greater than 2", bigDecimal); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java index 5e9275df3a1..07401096c87 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JavaResultPrinters.java @@ -60,7 +60,7 @@ public Object apply(@NotNull Collection value) { final List out = new ArrayList<>(value.size()); for (T elt : value) { - if (elt == null){ + if (elt == null) { // Printers do not handle null. continue; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java index 0841a78d45c..6501bea47de 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/JsonResultPrinters.java @@ -21,7 +21,10 @@ public Printer getDatePrinter(PrintSettings printSettings) { @Override public Printer> getDateRangePrinter(PrintSettings printSettings) { - return new DateRangeStringPrinter(printSettings, StringResultPrinters.NEGATIVE_INF_DEFAULT, StringResultPrinters.POSITIVE_INF_DEFAULT); + return new DateRangeStringPrinter( + printSettings, + StringResultPrinters.NEGATIVE_INF_DEFAULT, + StringResultPrinters.POSITIVE_INF_DEFAULT); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java index f1d377de6e9..afcb94db5d1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/Printer.java @@ -6,7 +6,7 @@ /** * Printers handle transformation from {@link com.bakdata.conquery.models.query.results.EntityResult} to the respective renderers "native" representation. - * + * * @param The intermediate representation of the type we are printing. */ @FunctionalInterface diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateRangeStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateRangeStringPrinter.java index 74c474d0b17..ee51ee92aac 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateRangeStringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/DateRangeStringPrinter.java @@ -11,8 +11,10 @@ @Slf4j public record DateRangeStringPrinter( - String negativeInf, String positiveInf, - DateStringPrinter datePrinter, PrintSettings cfg) implements Printer> { + String negativeInf, + String positiveInf, + DateStringPrinter datePrinter, + PrintSettings cfg) implements Printer> { public DateRangeStringPrinter(PrintSettings printSettings, String negativeInf, String positiveInf) { this(negativeInf, positiveInf, new DateStringPrinter(printSettings), printSettings); @@ -24,19 +26,24 @@ public String apply(@NotNull List f) { return null; } - Preconditions.checkArgument(f.size() == 2, "Expected a list with 2 elements, one min, one max. The list was: %s ", f); + Preconditions.checkArgument( + f.size() == 2, + "Expected a list with 2 elements, one min, one max. The list was: %s ", + f); final Integer min = f.get(0); final Integer max = f.get(1); // Compute minString first because we need it either way - final String minString = min == null || min == CDateRange.NEGATIVE_INFINITY ? negativeInf : datePrinter.apply(min); + final String minString = min == null || min == CDateRange.NEGATIVE_INFINITY ? negativeInf : datePrinter.apply( + min); if (cfg.isPrettyPrint() && min != null && min.equals(max)) { // If the min and max are the same we print it like a singe date, not a range (only in pretty printing) return minString; } - final String maxString = max == null || max == CDateRange.POSITIVE_INFINITY ? positiveInf : datePrinter.apply(max); + final String maxString = max == null || max == CDateRange.POSITIVE_INFINITY ? positiveInf : datePrinter.apply( + max); return minString + cfg.getDateRangeSeparator() + maxString; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java index 0e62677566b..4adadea5c0b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java @@ -9,7 +9,10 @@ import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import org.jetbrains.annotations.NotNull; -public record ListStringPrinter(Printer elementPrinter, PrintSettings cfg, LocaleConfig.ListFormat listFormat) implements Printer> { +public record ListStringPrinter( + Printer elementPrinter, + PrintSettings cfg, + LocaleConfig.ListFormat listFormat) implements Printer> { public ListStringPrinter(Printer elementPrinter, PrintSettings cfg) { this(elementPrinter, cfg, cfg.getListFormat()); @@ -21,10 +24,10 @@ public String apply(@NotNull Collection f) { final StringJoiner joiner = listFormat.createListJoiner(); for (T obj : f) { - if (obj == null){ + if (obj == null) { continue; } - + joiner.add(listFormat.escapeListElement(Objects.toString(elementPrinter.apply(obj)))); } return joiner.toString(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/LocalizedEnumPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/LocalizedEnumPrinter.java index 61cd8ccd60c..8b8e220c74c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/LocalizedEnumPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/LocalizedEnumPrinter.java @@ -5,13 +5,14 @@ import com.bakdata.conquery.models.query.resultinfo.printers.Printer; import org.jetbrains.annotations.NotNull; -public record LocalizedEnumPrinter & LocalizedToString>(PrintSettings cfg, Class clazz) implements Printer { +public record LocalizedEnumPrinter & LocalizedToString>( + PrintSettings cfg, + Class clazz) implements Printer { @Override public String apply(@NotNull String f) { try { return Enum.valueOf(clazz, f).toString(cfg.getLocale()); - } - catch (Exception e) { + } catch (Exception e) { throw new IllegalArgumentException("%s is not a valid %s.".formatted(f, clazz), e); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/OneToOneMappingPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/OneToOneMappingPrinter.java index 4495e3608f7..0748b5926c2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/OneToOneMappingPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/OneToOneMappingPrinter.java @@ -9,7 +9,7 @@ public record OneToOneMappingPrinter(InternToExternMapper mapper, Printer streamValues(); - + /** * Returns a list of the computed result line for this entity on the query. */ List listResultLines(); int length(); - + /** * Allows to modify the underlying result directly. The return value of the line modifier is the new line. - * So the modifier can change the array without reallocation by return the reference it received. Alternative it can - * allocate a new result line and return that reference instead. + * So the modifier can change the array without reallocation by return the reference it received. Alternative it can + * allocate a new result line and return that reference instead. * @param lineModifier A modifier(-chain) for a result line. */ void modifyResultLinesInplace(UnaryOperator lineModifier); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/results/FormShardResult.java b/backend/src/main/java/com/bakdata/conquery/models/query/results/FormShardResult.java index 965c812f248..55e8428d0da 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/results/FormShardResult.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/results/FormShardResult.java @@ -38,14 +38,17 @@ public FormShardResult(ManagedExecutionId formId, ManagedExecutionId subQueryId, */ @Override protected void addResult(DistributedExecutionManager executionManager) { - final ManagedInternalForm managedInternalForm = (ManagedInternalForm) executionManager.getExecution(getFormId()); + final ManagedInternalForm managedInternalForm = (ManagedInternalForm) executionManager.getExecution( + getFormId()); final ManagedQuery subQuery = managedInternalForm.getSubQuery(getExecutionId()); if (subQuery == null) { - throw new IllegalStateException("Subquery %s did not belong to form %s. Known subqueries: %s".formatted(getExecutionId(), - formId, - managedInternalForm.getSubQueries() - )); + throw new IllegalStateException( + "Subquery %s did not belong to form %s. Known subqueries: %s".formatted( + getExecutionId(), + formId, + managedInternalForm.getSubQueries() + )); } @@ -53,7 +56,11 @@ protected void addResult(DistributedExecutionManager executionManager) { // Fail the whole execution if a subquery fails if (ExecutionState.FAILED.equals(subQuery.getState())) { - managedInternalForm.fail(Optional.ofNullable(getError()).orElseThrow(() -> new IllegalStateException(String.format("Query[%s] failed but no error was set.", subQuery.getId())))); + managedInternalForm.fail( + Optional.ofNullable(getError()) + .orElseThrow( + () -> new IllegalStateException( + String.format("Query[%s] failed but no error was set.", subQuery.getId())))); } if (managedInternalForm.allSubQueriesDone()) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/results/MultilineEntityResult.java b/backend/src/main/java/com/bakdata/conquery/models/query/results/MultilineEntityResult.java index 2fb0442678f..cc461f7bc80 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/results/MultilineEntityResult.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/results/MultilineEntityResult.java @@ -1,22 +1,23 @@ package com.bakdata.conquery.models.query.results; +import jakarta.validation.constraints.NotNull; import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.function.UnaryOperator; import java.util.stream.Stream; -import jakarta.validation.constraints.NotNull; - import com.bakdata.conquery.io.cps.CPSType; import lombok.Getter; import lombok.Setter; import lombok.ToString; -@Getter @Setter @ToString -@CPSType(id="MULTI_LINE", base= EntityResult.class) +@Getter +@Setter +@ToString +@CPSType(id = "MULTI_LINE", base = EntityResult.class) public class MultilineEntityResult implements EntityResult { - + private final String entityId; @NotNull private final List values; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/results/ShardResult.java b/backend/src/main/java/com/bakdata/conquery/models/query/results/ShardResult.java index 815b75b552a..ac936cf5ce7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/results/ShardResult.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/results/ShardResult.java @@ -33,7 +33,7 @@ @Slf4j @ToString(onlyExplicitlyIncluded = true) @NoArgsConstructor(onConstructor_ = {@JsonCreator}) -public class ShardResult extends NamespaceMessage { +public class ShardResult extends NamespaceMessage { @ToString.Include @@ -68,14 +68,21 @@ public synchronized void finish(@NonNull List results, Worker work finishTime = LocalDateTime.now(); - log.info("FINISHED Query[{}] with {} results within {}", executionId, results.size(), Duration.between(startTime, finishTime)); + log.info( + "FINISHED Query[{}] with {} results within {}", + executionId, + results.size(), + Duration.between(startTime, finishTime)); this.results = results; // Truncate here because too large logs will crash/lock the process - log.trace("Collected Results for execution {}\n{}", executionId, StringUtils.truncate(results.toString(), 1000) + " (...)"); + log.trace( + "Collected Results for execution {}\n{}", + executionId, + StringUtils.truncate(results.toString(), 1000) + " (...)"); } - + protected void addResult(DistributedExecutionManager executionManager) { executionManager.handleQueryResult(this, ((ManagedQuery) executionManager.getExecution(executionId))); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/results/SinglelineEntityResult.java b/backend/src/main/java/com/bakdata/conquery/models/query/results/SinglelineEntityResult.java index 56a4c0cd56e..3a65e40e3ef 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/results/SinglelineEntityResult.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/results/SinglelineEntityResult.java @@ -13,8 +13,10 @@ import lombok.ToString; @ToString -@Getter @Setter @AllArgsConstructor(onConstructor_=@JsonCreator) -@CPSType(id="SINGLE_LINE", base= EntityResult.class) +@Getter +@Setter +@AllArgsConstructor(onConstructor_ = @JsonCreator) +@CPSType(id = "SINGLE_LINE", base = EntityResult.class) public class SinglelineEntityResult implements EntityResult { private final String entityId; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/BooleanColumnStatsCollector.java b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/BooleanColumnStatsCollector.java index fd4fb70237e..6561575c957 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/BooleanColumnStatsCollector.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/BooleanColumnStatsCollector.java @@ -29,8 +29,7 @@ public void consume(Object value) { if (((Boolean) value)) { trues++; - } - else { + } else { falses++; } } @@ -40,15 +39,17 @@ public ResultColumnStatistics describe() { final Results results = C10nCache.getLocalized(Results.class, getPrintSettings().getLocale()); return new HistogramColumnDescription( - getName(), getLabel(), getDescription(), - List.of( - new HistogramColumnDescription.Entry(results.True(), trues), - new HistogramColumnDescription.Entry(results.False(), falses) - ), - Map.of( - C10N.get(StatisticsLabels.class, getPrintSettings().getLocale()).missing(), - getPrintSettings().getIntegerFormat().format(getMissing()) - ) + getName(), + getLabel(), + getDescription(), + List.of( + new HistogramColumnDescription.Entry(results.True(), trues), + new HistogramColumnDescription.Entry(results.False(), falses) + ), + Map.of( + C10N.get(StatisticsLabels.class, getPrintSettings().getLocale()).missing(), + getPrintSettings().getIntegerFormat().format(getMissing()) + ) ); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ColumnStatsCollector.java b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ColumnStatsCollector.java index 689f2e11294..640646e2761 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ColumnStatsCollector.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ColumnStatsCollector.java @@ -18,20 +18,43 @@ public abstract class ColumnStatsCollector { @JsonIgnore private final PrintSettings printSettings; - public static ColumnStatsCollector getStatsCollector(String name, String description, ResultType type, PrintSettings printSettings, FrontendConfig config) { + public static ColumnStatsCollector getStatsCollector( + String name, + String description, + ResultType type, + PrintSettings printSettings, + FrontendConfig config) { // List recursion must be done before assigning uniqueNames if (type instanceof ResultType.ListT listT) { - final ColumnStatsCollector columnStatsCollector = getStatsCollector(name, description, listT.getElementType(), printSettings, config); + final ColumnStatsCollector columnStatsCollector = getStatsCollector( + name, + description, + listT.getElementType(), + printSettings, + config); return new ListColumnStatsCollector(columnStatsCollector, printSettings); } return switch (((ResultType.Primitive) type)) { case BOOLEAN -> new BooleanColumnStatsCollector(name, name, description, printSettings); - case INTEGER, MONEY, NUMERIC -> new NumberColumnStatsCollector<>(name, name, description, type, printSettings, config.getVisualisationsHistogramLimit(), config.getVisualisationPercentiles().lowerEndpoint(), config.getVisualisationPercentiles().upperEndpoint()); + case INTEGER, MONEY, NUMERIC -> new NumberColumnStatsCollector<>( + name, + name, + description, + type, + printSettings, + config.getVisualisationsHistogramLimit(), + config.getVisualisationPercentiles().lowerEndpoint(), + config.getVisualisationPercentiles().upperEndpoint()); case DATE, DATE_RANGE -> new DateColumnStatsCollector(name, name, description, type, printSettings); - case STRING -> new StringColumnStatsCollector(name, name, description, printSettings, config.getVisualisationsHistogramLimit()); + case STRING -> new StringColumnStatsCollector( + name, + name, + description, + printSettings, + config.getVisualisationsHistogramLimit()); }; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/DateColumnStatsCollector.java b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/DateColumnStatsCollector.java index 2c661d5f3d1..fb73e8db201 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/DateColumnStatsCollector.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/DateColumnStatsCollector.java @@ -27,7 +27,12 @@ public class DateColumnStatsCollector extends ColumnStatsCollector { private CDateRange span = null; - public DateColumnStatsCollector(String name, String label, String description, ResultType type, PrintSettings printSettings) { + public DateColumnStatsCollector( + String name, + String label, + String description, + ResultType type, + PrintSettings printSettings) { super(name, label, description, printSettings); dateExtractor = getDateExtractor(type); } @@ -76,11 +81,14 @@ private void handleDay(int day) { @Override public ResultColumnStatistics describe() { - return new ColumnDescription(getName(), getLabel(), getDescription(), - totalCount, - nulls, - new TreeMap<>(monthCounts), - span == null ? CDateRange.all().toSimpleRange() : span.toSimpleRange() + return new ColumnDescription( + getName(), + getLabel(), + getDescription(), + totalCount, + nulls, + new TreeMap<>(monthCounts), + span == null ? CDateRange.all().toSimpleRange() : span.toSimpleRange() ); } @@ -95,7 +103,14 @@ public static class ColumnDescription extends ResultColumnStatistics { private final Range span; - public ColumnDescription(String name, String label, String description, int count, int nullValues, SortedMap monthCounts, Range span) { + public ColumnDescription( + String name, + String label, + String description, + int count, + int nullValues, + SortedMap monthCounts, + Range span) { super(name, label, description); this.count = count; this.nullValues = nullValues; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/Histogram.java b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/Histogram.java index d4725e79449..68d20d6d8e8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/Histogram.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/Histogram.java @@ -6,7 +6,6 @@ import com.google.common.collect.Range; import com.google.common.collect.TreeRangeMap; import it.unimi.dsi.fastutil.doubles.Double2ObjectFunction; -import it.unimi.dsi.fastutil.doubles.Double2ObjectMaps; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; @@ -44,7 +43,14 @@ protected Histogram(TreeRangeMap nodes, double absMin, double a /** * Create a histogram that is segmented to always have 0 singled out. */ - public static Histogram zeroAligned(double lower, double upper, double absMin, double absMax, int expectedBins, boolean roundWidth, boolean integral) { + public static Histogram zeroAligned( + double lower, + double upper, + double absMin, + double absMax, + int expectedBins, + boolean roundWidth, + boolean integral) { if (lower == upper) { TreeRangeMap nodes = TreeRangeMap.create(); @@ -65,13 +71,18 @@ public static Histogram zeroAligned(double lower, double upper, double absMin, d if (roundWidth) { return rounded(lower, upper, absMin, absMax, expectedBins, integral); - } - else { + } else { return unrounded(lower, upper, absMin, absMax, expectedBins, integral); } } - private static Histogram rounded(double lower, double upper, double absMin, double absMax, int expectedBins, boolean integral) { + private static Histogram rounded( + double lower, + double upper, + double absMin, + double absMax, + int expectedBins, + boolean integral) { // adjust lower/upper to start on rounded edges. double adjLower = Math.max(Math.floor(absMin), Math.floor(lower)); final double adjUpper = Math.min(Math.ceil(absMax), Math.ceil(upper)); @@ -81,14 +92,27 @@ private static Histogram rounded(double lower, double upper, double absMin, doub return createHistogram(lower, absMin, absMax, expectedBins, adjLower, binWidth, integral); } - private static Histogram unrounded(double lower, double upper, double absMin, double absMax, int expectedBins, boolean integral) { + private static Histogram unrounded( + double lower, + double upper, + double absMin, + double absMax, + int expectedBins, + boolean integral) { double binWidth = (upper - lower) / expectedBins; return createHistogram(lower, absMin, absMax, expectedBins, lower, binWidth, integral); } @NotNull - private static Histogram createHistogram(double lower, double absMin, double absMax, int expectedBins, double adjLower, double binWidth, boolean integral) { + private static Histogram createHistogram( + double lower, + double absMin, + double absMax, + int expectedBins, + double adjLower, + double binWidth, + boolean integral) { adjLower = ensureZeroCrossing(lower, absMax, adjLower, binWidth); final TreeRangeMap nodes = TreeRangeMap.create(); @@ -96,9 +120,11 @@ private static Histogram createHistogram(double lower, double absMin, double abs // Note, using multiplication is important to avoid floating-point imprecision when wanting to arrive exactly around 0 etc. for (int index = 0; index < expectedBins; index++) { Counter node = new Counter(); - nodes.put(Range.closedOpen( + nodes.put( + Range.closedOpen( adjLower + binWidth * index, - adjLower + binWidth * (index + 1)), node); + adjLower + binWidth * (index + 1)), + node); } Range span = nodes.span(); @@ -141,9 +167,7 @@ public String createLabel(Range node, Double2ObjectFunction prin return lower; } - final String upper = node.hasUpperBound() - ? printer.apply(max) - : printer.apply(max - 1); + final String upper = node.hasUpperBound() ? printer.apply(max) : printer.apply(max - 1); return lower + FROM_TO + upper; } @@ -173,17 +197,14 @@ public void add(double value) { } public List, Counter>> nodes() { - return nodes.asMapOfRanges().entrySet().stream() - .filter(entry -> { - Range span = entry.getKey(); - if (span.hasLowerBound() && span.hasUpperBound()) { - return true; - } - else { - return entry.getValue().getCount() > 0; - } - }) - .toList(); + return nodes.asMapOfRanges().entrySet().stream().filter(entry -> { + Range span = entry.getKey(); + if (span.hasLowerBound() && span.hasUpperBound()) { + return true; + } else { + return entry.getValue().getCount() > 0; + } + }).toList(); } @Data diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/HistogramColumnDescription.java b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/HistogramColumnDescription.java index 9106859f8a8..3a86faa3b1b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/HistogramColumnDescription.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/HistogramColumnDescription.java @@ -12,12 +12,19 @@ @ToString(callSuper = true) public class HistogramColumnDescription extends ColumnStatsCollector.ResultColumnStatistics { - public record Entry(String label, long value) {}; + public record Entry(String label, long value) { + } + private final List entries; private final Map extras; - public HistogramColumnDescription(String name, String label, String description, List histogram, Map extras) { + public HistogramColumnDescription( + String name, + String label, + String description, + List histogram, + Map extras) { super(name, label, description); this.entries = histogram; this.extras = extras; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ListColumnStatsCollector.java b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ListColumnStatsCollector.java index a274baf448d..2a095f056d7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ListColumnStatsCollector.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ListColumnStatsCollector.java @@ -21,7 +21,7 @@ public ListColumnStatsCollector(ColumnStatsCollector underlying, PrintSettings p @Override public void consume(@Nullable Object values) { - if(values == null){ + if (values == null) { return; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/NumberColumnStatsCollector.java b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/NumberColumnStatsCollector.java index fb4553dc913..efbb3151c04 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/NumberColumnStatsCollector.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/NumberColumnStatsCollector.java @@ -32,14 +32,14 @@ public class NumberColumnStatsCollector> private int nulls; public NumberColumnStatsCollector( - String name, - String label, - String description, - ResultType type, - PrintSettings printSettings, - int expectedBins, - double lowerPercentile, - double upperPercentile) { + String name, + String label, + String description, + ResultType type, + PrintSettings printSettings, + int expectedBins, + double lowerPercentile, + double upperPercentile) { super(name, label, description, printSettings); this.type = type; @@ -61,7 +61,8 @@ private Comparator selectComparator(ResultType resultType) { return switch (((ResultType.Primitive) type)) { case INTEGER -> Comparator.comparingInt(Number::intValue); case MONEY, NUMERIC -> Comparator.comparingDouble(Number::doubleValue); - default -> throw new IllegalArgumentException("Cannot handle result type %s".formatted(resultType.toString())); + default -> + throw new IllegalArgumentException("Cannot handle result type %s".formatted(resultType.toString())); }; } @@ -76,7 +77,12 @@ private static NumberFormat selectFormatter(ResultType type, PrintSettings print /** * If distance between bounds is less than expectedBins, we expand our bounds along percentiles. */ - private static Range expandBounds(double lower, double upper, int expectedBins, DescriptiveStatistics statistics, double by) { + private static Range expandBounds( + double lower, + double upper, + int expectedBins, + DescriptiveStatistics statistics, + double by) { assert by > 0; @@ -113,7 +119,12 @@ public void consume(Object value) { public ResultColumnStatistics describe() { // If no real samples were collected, we short-circuit, as Statistics will throw an exception when empty. if (getStatistics().getN() == 0) { - return new HistogramColumnDescription(getName(), getLabel(), getDescription(), Collections.emptyList(), Collections.emptyMap()); + return new HistogramColumnDescription( + getName(), + getLabel(), + getDescription(), + Collections.emptyList(), + Collections.emptyMap()); } final List bins = createBins(); @@ -129,43 +140,41 @@ private List createBins() { final Range bounds; // Shortcut for ranges close to exactly [0..1] - if (statistics.getMin() > 0 && statistics.getMin() <= 0.1 && statistics.getMax() < 1 && statistics.getMax() > 0.9) { + if (statistics.getMin() > 0 && statistics.getMin() <= 0.1 && statistics.getMax() < 1 && statistics + .getMax() > 0.9) { bounds = Range.closed(0d, 1d); - } - else { + } else { bounds = expandBounds(lowerPercentile, upperPercentile, expectedBins, statistics, 5); } - log.trace("Creating Histogram for {} with params inner=({}, {}), bounds=({},{}) bins={}", - getLabel(), - bounds.lowerEndpoint(), - bounds.upperEndpoint(), - getStatistics().getMin(), - getStatistics().getMax(), - expectedBins + log.trace( + "Creating Histogram for {} with params inner=({}, {}), bounds=({},{}) bins={}", + getLabel(), + bounds.lowerEndpoint(), + bounds.upperEndpoint(), + getStatistics().getMin(), + getStatistics().getMax(), + expectedBins ); boolean integral = ResultType.Primitive.INTEGER.equals(getType()); - final Histogram histogram = - Histogram.zeroAligned(bounds.lowerEndpoint(), - bounds.upperEndpoint(), - getStatistics().getMin(), - getStatistics().getMax(), - expectedBins, - integral || bounds.upperEndpoint() - bounds.lowerEndpoint() > 1, - integral - ); + final Histogram histogram = Histogram.zeroAligned( + bounds.lowerEndpoint(), + bounds.upperEndpoint(), + getStatistics().getMin(), + getStatistics().getMax(), + expectedBins, + integral || bounds.upperEndpoint() - bounds.lowerEndpoint() > 1, + integral + ); Arrays.stream(getStatistics().getValues()).forEach(histogram::add); - return histogram.getNodes().asMapOfRanges().entrySet() - .stream() - .map(bin -> { - final String binLabel = histogram.createLabel(bin.getKey(), this::printValue); + return histogram.getNodes().asMapOfRanges().entrySet().stream().map(bin -> { + final String binLabel = histogram.createLabel(bin.getKey(), this::printValue); - return new HistogramColumnDescription.Entry(binLabel, bin.getValue().getCount()); - }) - .toList(); + return new HistogramColumnDescription.Entry(binLabel, bin.getValue().getCount()); + }).toList(); } @NotNull @@ -182,8 +191,7 @@ private Map getExtras() { // mean is always a decimal number, therefore integer needs special handling if (ResultType.Primitive.INTEGER.equals(getType())) { out.put(labels.mean(), decimalFormat.format(getStatistics().getMean())); - } - else { + } else { out.put(labels.mean(), printValue(getStatistics().getMean())); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ResultStatistics.java b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ResultStatistics.java index 26746e58407..0c113fa5ad5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ResultStatistics.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/ResultStatistics.java @@ -1,5 +1,16 @@ package com.bakdata.conquery.models.query.statistics; +import java.time.LocalDate; +import java.util.List; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.Callable; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.stream.IntStream; + import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.config.ConqueryConfig; @@ -22,35 +33,28 @@ import org.apache.commons.lang3.time.StopWatch; import org.jetbrains.annotations.NotNull; -import java.time.LocalDate; -import java.util.List; -import java.util.Optional; -import java.util.OptionalLong; -import java.util.concurrent.Callable; -import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.BiConsumer; -import java.util.function.Consumer; -import java.util.stream.IntStream; - @Slf4j -public record ResultStatistics(int entities, int total, List statistics, - Range dateRange) { +public record ResultStatistics( + int entities, + int total, + List statistics, + Range dateRange) { @SneakyThrows @NotNull public static ResultStatistics collectResultStatistics( - SingleTableResult managedQuery, - List resultInfos, - Optional dateInfo, - Optional dateIndex, - PrintSettings printSettings, - UniqueNamer uniqueNamer, - ConqueryConfig conqueryConfig, - PrinterFactory printerFactory) { + SingleTableResult managedQuery, + List resultInfos, + Optional dateInfo, + Optional dateIndex, + PrintSettings printSettings, + UniqueNamer uniqueNamer, + ConqueryConfig conqueryConfig, + PrinterFactory printerFactory) { //TODO pull inner executor service from ManagerNode - final ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(Math.max(1, Runtime.getRuntime().availableProcessors() - 1))); + final ListeningExecutorService executorService = MoreExecutors.listeningDecorator( + Executors.newFixedThreadPool(Math.max(1, Runtime.getRuntime().availableProcessors() - 1))); // Yes, we are actually iterating the result for every job. @@ -60,62 +64,73 @@ public static ResultStatistics collectResultStatistics( final boolean containsDates = dateInfo.isPresent() && dateIndex.isPresent(); if (containsDates) { - futureSpan = executorService.submit(() -> calculateDateSpan(managedQuery, dateInfo, dateIndex.get(), printSettings)); + futureSpan = executorService.submit( + () -> calculateDateSpan(managedQuery, dateInfo, dateIndex.get(), printSettings)); } else { futureSpan = Futures.immediateFuture(CDateRange.all().toSimpleRange()); } // Count result lines and entities (may differ in case of form or SecondaryIdQuery) - final ListenableFuture futureLines = executorService.submit(() -> (int) managedQuery.resultRowCount().orElse(0)); + final ListenableFuture futureLines = executorService.submit( + () -> (int) managedQuery.resultRowCount().orElse(0)); - final ListenableFuture futureEntities = executorService.submit(() -> (int) managedQuery.streamResults(OptionalLong.empty()).count()); + final ListenableFuture futureEntities = executorService.submit( + () -> (int) managedQuery.streamResults(OptionalLong.empty()).count()); // compute ResultColumnStatistics for each column - final List> futureDescriptions = - IntStream.range(0, resultInfos.size()) - // If the query doesn't contain dates, we can skip the dates-column. - .filter(col -> !resultInfos.get(col).getSemantics().contains(new SemanticType.EventDateT()) || containsDates) - .mapToObj(col -> (Callable) () -> { - final StopWatch started = StopWatch.createStarted(); - - final ResultInfo info = resultInfos.get(col); - final Printer printer = info.createPrinter(printerFactory, printSettings); - final ColumnStatsCollector statsCollector = - ColumnStatsCollector.getStatsCollector(uniqueNamer.getUniqueName(info, printSettings), - info.getDescription(), - info.getType(), - printSettings, - conqueryConfig.getFrontend() - ); - - log.trace("BEGIN stats collection for {}", info); - - managedQuery.streamResults(OptionalLong.empty()) - .map(EntityResult::listResultLines) - .flatMap(List::stream) - .forEach(line -> { - final Object value = line[col]; - if (value == null) { - // Printers dont handle null - statsCollector.consume(null); - return; - } - statsCollector.consume(printer.apply(value)); - }); - - log.trace("DONE collecting values for {}, in {}", info, started); - - final ColumnStatsCollector.ResultColumnStatistics description = statsCollector.describe(); - - log.debug("DONE description for {}, in {}", info, started); - - return description; - }) - .map(executorService::submit) - .toList(); + final List> futureDescriptions = IntStream.range( + 0, + resultInfos.size()) + // If the query doesn't contain dates, we can skip the dates-column. + .filter( + col -> !resultInfos.get(col) + .getSemantics() + .contains( + new SemanticType.EventDateT()) || containsDates) + .mapToObj( + col -> (Callable) () -> { + final StopWatch started = StopWatch.createStarted(); + + final ResultInfo info = resultInfos.get(col); + final Printer printer = info.createPrinter(printerFactory, printSettings); + final ColumnStatsCollector statsCollector = ColumnStatsCollector.getStatsCollector( + uniqueNamer.getUniqueName(info, printSettings), + info.getDescription(), + info.getType(), + printSettings, + conqueryConfig.getFrontend() + ); + + log.trace("BEGIN stats collection for {}", info); + + managedQuery.streamResults(OptionalLong.empty()) + .map( + EntityResult::listResultLines) + .flatMap(List::stream) + .forEach(line -> { + final Object value = line[col]; + if (value == null) { + // Printers dont handle null + statsCollector.consume(null); + return; + } + statsCollector.consume(printer.apply(value)); + }); + + log.trace("DONE collecting values for {}, in {}", info, started); + + final ColumnStatsCollector.ResultColumnStatistics description = statsCollector.describe(); + + log.debug("DONE description for {}, in {}", info, started); + + return description; + }) + .map(executorService::submit) + .toList(); final Range span = futureSpan.get(); - final List descriptions = Futures.allAsList(futureDescriptions).get(); + final List descriptions = Futures.allAsList( + futureDescriptions).get(); final int lines = futureLines.get(); final int entities = futureEntities.get(); @@ -124,7 +139,11 @@ public static ResultStatistics collectResultStatistics( return new ResultStatistics(entities, lines, descriptions, span); } - private static Range calculateDateSpan(SingleTableResult managedQuery, Optional dateInfo, int dateIndex, PrintSettings printSettings) { + private static Range calculateDateSpan( + SingleTableResult managedQuery, + Optional dateInfo, + int dateIndex, + PrintSettings printSettings) { if (dateInfo.isEmpty()) { return CDateRange.all().toSimpleRange(); } @@ -146,18 +165,30 @@ private static Range calculateDateSpan(SingleTableResult managedQuery /** * If not dateInfo is given, don't try to span values. otherwise takes values from line at dateIndex, and handles them according to dateInfo. */ - private static Consumer getDateSpanner(ResultInfo dateInfo, int dateIndex, AtomicReference spanRef, PrintSettings printSettings) { + private static Consumer getDateSpanner( + ResultInfo dateInfo, + int dateIndex, + AtomicReference spanRef, + PrintSettings printSettings) { - final Consumer spanner = date -> spanRef.getAndAccumulate(date, (old, incoming) -> incoming.spanClosed(old)); + final Consumer spanner = date -> spanRef.getAndAccumulate( + date, + (old, incoming) -> incoming.spanClosed(old)); final JavaResultPrinters printers = new JavaResultPrinters(); - final BiConsumer> extractor = validityDateExtractor(dateInfo.getType(), printSettings, printers); + final BiConsumer> extractor = validityDateExtractor( + dateInfo.getType(), + printSettings, + printers); return line -> extractor.accept(line[dateIndex], spanner); } - public static BiConsumer> validityDateExtractor(ResultType dateType, PrintSettings printSettings, JavaResultPrinters printers) { + public static BiConsumer> validityDateExtractor( + ResultType dateType, + PrintSettings printSettings, + JavaResultPrinters printers) { if (dateType.equals(ResultType.Primitive.DATE_RANGE)) { final Printer printer = printers.getDateRangePrinter(printSettings); @@ -171,7 +202,10 @@ public static BiConsumer> validityDateExtractor(Res } if (dateType instanceof ResultType.ListT listT) { - final BiConsumer> extractor = validityDateExtractor(listT.getElementType(), printSettings, printers); + final BiConsumer> extractor = validityDateExtractor( + listT.getElementType(), + printSettings, + printers); return (obj, con) -> ((List) obj).forEach(date -> extractor.accept(date, con)); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/StringColumnStatsCollector.java b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/StringColumnStatsCollector.java index c66d368b65c..5da2516eb29 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/statistics/StringColumnStatsCollector.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/statistics/StringColumnStatsCollector.java @@ -22,7 +22,12 @@ public class StringColumnStatsCollector extends ColumnStatsCollector { private int nulls = 0; - public StringColumnStatsCollector(String name, String label, String description, PrintSettings printSettings, long limit) { + public StringColumnStatsCollector( + String name, + String label, + String description, + PrintSettings printSettings, + long limit) { super(name, label, description, printSettings); this.limit = limit; } @@ -41,10 +46,9 @@ public void consume(Object value) { @Override public ResultColumnStatistics describe() { - final List, Long>> entriesSorted = - StreamSupport.stream(((Iterable, Long>>) frequencies::entrySetIterator).spliterator(), false) - .sorted(Map.Entry., Long>comparingByValue().reversed()) - .toList(); + final List, Long>> entriesSorted = StreamSupport.stream( + ((Iterable, Long>>) frequencies::entrySetIterator).spliterator(), + false).sorted(Map.Entry., Long>comparingByValue().reversed()).toList(); final long end = Math.min(limit, entriesSorted.size()); @@ -54,7 +58,9 @@ public ResultColumnStatistics describe() { for (int i = 0; i < end; i++) { final Map.Entry, Long> counts = entriesSorted.get(i); - final HistogramColumnDescription.Entry entry = new HistogramColumnDescription.Entry(((String) counts.getKey()), counts.getValue()); + final HistogramColumnDescription.Entry entry = new HistogramColumnDescription.Entry( + ((String) counts.getKey()), + counts.getValue()); head.add(entry); shownTotal += counts.getValue(); @@ -67,8 +73,8 @@ public ResultColumnStatistics describe() { if (entriesSorted.size() > limit) { extras.put( - statisticsLabels.remainingValues(entriesSorted.size() - limit), - statisticsLabels.remainingEntries(frequencies.getSumFreq() - shownTotal) + statisticsLabels.remainingValues(entriesSorted.size() - limit), + statisticsLabels.remainingEntries(frequencies.getSumFreq() - shownTotal) ); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java b/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java index c49969a82c1..e6397ddbb18 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java +++ b/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java @@ -28,79 +28,133 @@ public static ResultType resolveResultType(MajorTypeId majorTypeId) { String typeInfo(); - T getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException; + T getFromResultSet( + ResultSet resultSet, + int columnIndex, + ResultSetProcessor resultSetProcessor) throws SQLException; - List getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException; + List getFromResultSetAsList( + ResultSet resultSet, + int columnIndex, + ResultSetProcessor resultSetProcessor) throws SQLException; enum Primitive implements ResultType { BOOLEAN { @Override - public Boolean getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { + public Boolean getFromResultSet( + ResultSet resultSet, + int columnIndex, + ResultSetProcessor resultSetProcessor) throws SQLException { return resultSetProcessor.getBoolean(resultSet, columnIndex); } @Override - public List getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { + public List getFromResultSetAsList( + ResultSet resultSet, + int columnIndex, + ResultSetProcessor resultSetProcessor) throws SQLException { return resultSetProcessor.getBooleanList(resultSet, columnIndex); } - }, INTEGER { + }, + INTEGER { @Override - public Integer getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { + public Integer getFromResultSet( + ResultSet resultSet, + int columnIndex, + ResultSetProcessor resultSetProcessor) throws SQLException { return resultSetProcessor.getInteger(resultSet, columnIndex); } @Override - public List getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { + public List getFromResultSetAsList( + ResultSet resultSet, + int columnIndex, + ResultSetProcessor resultSetProcessor) throws SQLException { return resultSetProcessor.getIntegerList(resultSet, columnIndex); } - }, NUMERIC { + }, + NUMERIC { @Override - public Double getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { + public Double getFromResultSet( + ResultSet resultSet, + int columnIndex, + ResultSetProcessor resultSetProcessor) throws SQLException { return resultSetProcessor.getDouble(resultSet, columnIndex); } @Override - public List getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { + public List getFromResultSetAsList( + ResultSet resultSet, + int columnIndex, + ResultSetProcessor resultSetProcessor) throws SQLException { return resultSetProcessor.getDoubleList(resultSet, columnIndex); } - }, DATE { + }, + DATE { @Override - public Number getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { + public Number getFromResultSet( + ResultSet resultSet, + int columnIndex, + ResultSetProcessor resultSetProcessor) throws SQLException { return resultSetProcessor.getDate(resultSet, columnIndex); } @Override - public List getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { + public List getFromResultSetAsList( + ResultSet resultSet, + int columnIndex, + ResultSetProcessor resultSetProcessor) throws SQLException { return resultSetProcessor.getDateList(resultSet, columnIndex); } - }, DATE_RANGE { + }, + DATE_RANGE { @Override - public List getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { + public List getFromResultSet( + ResultSet resultSet, + int columnIndex, + ResultSetProcessor resultSetProcessor) throws SQLException { return resultSetProcessor.getDateRange(resultSet, columnIndex); } @Override - public List> getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { + public List> getFromResultSetAsList( + ResultSet resultSet, + int columnIndex, + ResultSetProcessor resultSetProcessor) throws SQLException { return resultSetProcessor.getDateRangeList(resultSet, columnIndex); } - }, STRING { + }, + STRING { @Override - public String getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { + public String getFromResultSet( + ResultSet resultSet, + int columnIndex, + ResultSetProcessor resultSetProcessor) throws SQLException { return resultSetProcessor.getString(resultSet, columnIndex); } @Override - public List getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { + public List getFromResultSetAsList( + ResultSet resultSet, + int columnIndex, + ResultSetProcessor resultSetProcessor) throws SQLException { return resultSetProcessor.getStringList(resultSet, columnIndex); } - }, MONEY { + }, + MONEY { @Override - public BigDecimal getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { + public BigDecimal getFromResultSet( + ResultSet resultSet, + int columnIndex, + ResultSetProcessor resultSetProcessor) throws SQLException { return resultSetProcessor.getMoney(resultSet, columnIndex); } @Override - public List getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { + public List getFromResultSetAsList( + ResultSet resultSet, + int columnIndex, + ResultSetProcessor resultSetProcessor) throws SQLException { return resultSetProcessor.getMoneyList(resultSet, columnIndex); } }; @@ -131,12 +185,18 @@ public ListT(@NonNull ResultType elementType) { } @Override - public List getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { + public List getFromResultSet( + ResultSet resultSet, + int columnIndex, + ResultSetProcessor resultSetProcessor) throws SQLException { return elementType.getFromResultSetAsList(resultSet, columnIndex, resultSetProcessor); } @Override - public List> getFromResultSetAsList(final ResultSet resultSet, final int columnIndex, final ResultSetProcessor resultSetProcessor) { + public List> getFromResultSetAsList( + final ResultSet resultSet, + final int columnIndex, + final ResultSetProcessor resultSetProcessor) { throw new UnsupportedOperationException("Nested lists not supported in SQL mode"); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/ClusterHealthCheck.java b/backend/src/main/java/com/bakdata/conquery/models/worker/ClusterHealthCheck.java index 30292b2b49d..8f7818ec2a1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/ClusterHealthCheck.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/ClusterHealthCheck.java @@ -24,28 +24,36 @@ public class ClusterHealthCheck extends HealthCheck { protected Result check() throws Exception { Collection knownShards = clusterState.getShardNodes().values(); - final List disconnectedWorkers = - knownShards.stream() - .filter(Predicate.not(ShardNodeInformation::isConnected)) - .map(ShardNodeInformation::toString) - .toList(); + final List disconnectedWorkers = knownShards.stream() + .filter( + Predicate.not(ShardNodeInformation::isConnected)) + .map(ShardNodeInformation::toString) + .toList(); if (!disconnectedWorkers.isEmpty()) { - return Result.unhealthy("The shard(s) %s are no longer connected.".formatted(String.join(",", disconnectedWorkers))); + return Result.unhealthy( + "The shard(s) %s are no longer connected.".formatted(String.join(",", disconnectedWorkers))); } LocalDateTime now = LocalDateTime.now(); - List timeoutShards = - clusterState.getShardNodes().values().stream() - .filter((status) -> heartbeatTimeout.minus(Duration.between(status.getLastStatusTime(), now).abs()) - .isNegative()).toList(); + List timeoutShards = clusterState.getShardNodes() + .values() + .stream() + .filter( + (status) -> heartbeatTimeout.minus( + Duration.between(status.getLastStatusTime(), now).abs()).isNegative()) + .toList(); if (!timeoutShards.isEmpty()) { - return Result.unhealthy("Shards timed out:%s".formatted(timeoutShards.stream() - .map(ShardNodeInformation::getSession) - .map(NetworkSession::getRemoteAddress) - .map(SocketAddress::toString) - .collect(Collectors.joining(", ")))); + return Result.unhealthy( + "Shards timed out:%s".formatted( + timeoutShards.stream() + .map(ShardNodeInformation::getSession) + .map( + NetworkSession::getRemoteAddress) + .map(SocketAddress::toString) + .collect( + Collectors.joining(", ")))); } return Result.healthy(HEALTHY_MESSAGE_FMT, knownShards.size()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java b/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java index e611a976b11..9da9baeb629 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java @@ -52,7 +52,9 @@ public class DatasetRegistry implements Closeable, Namespac public N createNamespace(Dataset dataset, MetaStorage metaStorage, Environment environment) throws IOException { // Prepare empty storage NamespaceStorage datasetStorage = new NamespaceStorage(config.getStorage(), "dataset_" + dataset.getName()); - final ObjectMapper persistenceMapper = internalMapperFactory.createNamespacePersistenceMapper(datasetStorage, this); + final ObjectMapper persistenceMapper = internalMapperFactory.createNamespacePersistenceMapper( + datasetStorage, + this); dataset.setStorageProvider(this); // Each store injects its own IdResolveCtx so each needs its own mapper @@ -66,13 +68,13 @@ public N createNamespace(Dataset dataset, MetaStorage metaStorage, Environment e datasetStorage.close(); return createNamespace(datasetStorage, metaStorage, environment); - }catch (Exception e) { + } catch (Exception e) { log.error("Error while creating namespace for dataset {}. Deleting.", dataset.getId(), e); datasetStorage.openStores(Jackson.copyMapperAndInjectables(persistenceMapper)); datasetStorage.removeStorage(); - throw e; + throw e; } } @@ -100,9 +102,11 @@ public void removeNamespace(DatasetId id) { } public Stream getAllDatasets() { - return datasets.values().stream() - .map(Namespace::getStorage) - .map(namespaceStorage -> namespaceStorage.getDataset().getId()); + return datasets.values() + .stream() + .map(Namespace::getStorage) + .map( + namespaceStorage -> namespaceStorage.getDataset().getId()); } public Collection getNamespaces() { @@ -122,8 +126,7 @@ public void close() { for (Namespace namespace : datasets.values()) { try { namespace.close(); - } - catch (Exception e) { + } catch (Exception e) { log.error("Unable to close namespace {}", namespace, e); } } @@ -133,8 +136,7 @@ public void close() { public MutableInjectableValues inject(MutableInjectableValues values) { indexService.inject(values); // Make this class also available under DatasetRegistry - return values.add(NamespacedStorageProvider.class, this) - .add(DatasetRegistry.class, this); + return values.add(NamespacedStorageProvider.class, this).add(DatasetRegistry.class, this); } public void resetIndexService() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java index 2ca5255a718..8216308dac6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java @@ -1,4 +1,11 @@ package com.bakdata.conquery.models.worker; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.mode.cluster.ClusterEntityResolver; import com.bakdata.conquery.models.config.ClusterConfig; @@ -17,13 +24,6 @@ import lombok.ToString; import lombok.extern.slf4j.Slf4j; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - /** * Keep track of all data assigned to a single dataset. Each ShardNode has one {@link Worker} per {@link Dataset} / {@link DistributedNamespace}. @@ -39,14 +39,14 @@ public class DistributedNamespace extends Namespace { private final ClusterConfig clusterConfig; public DistributedNamespace( - ObjectMapper preprocessMapper, - NamespaceStorage storage, - DistributedExecutionManager executionManager, - JobManager jobManager, - SearchProcessor filterSearch, - ClusterEntityResolver clusterEntityResolver, - WorkerHandler workerHandler, - ClusterConfig clusterConfig) { + ObjectMapper preprocessMapper, + NamespaceStorage storage, + DistributedExecutionManager executionManager, + JobManager jobManager, + SearchProcessor filterSearch, + ClusterEntityResolver clusterEntityResolver, + WorkerHandler workerHandler, + ClusterConfig clusterConfig) { super(preprocessMapper, storage, executionManager, jobManager, filterSearch, clusterEntityResolver); this.executionManager = executionManager; this.workerHandler = workerHandler; @@ -55,11 +55,9 @@ public DistributedNamespace( @Override void updateMatchingStats() { - try(Stream> allConcepts = getStorage().getAllConcepts()) { - final Collection concepts = allConcepts - .filter(concept -> concept.getMatchingStats() == null) - .map(Concept::getId) - .collect(Collectors.toSet()); + try (Stream> allConcepts = getStorage().getAllConcepts()) { + final Collection concepts = allConcepts.filter( + concept -> concept.getMatchingStats() == null).map(Concept::getId).collect(Collectors.toSet()); getWorkerHandler().sendToAll(new UpdateMatchingStatsMessage(concepts)); } } @@ -69,8 +67,9 @@ void registerColumnValuesInSearch(Set columns) { log.trace("Sending columns to collect values on shards: {}", Arrays.toString(columns.toArray())); final CollectColumnValuesMessage columnValuesJob = new CollectColumnValuesMessage( - clusterConfig.getColumnValuesPerChunk(), - columns.stream().map(Column::getId).collect(Collectors.toSet()), this + clusterConfig.getColumnValuesPerChunk(), + columns.stream().map(Column::getId).collect(Collectors.toSet()), + this ); getWorkerHandler().sendToAll(columnValuesJob); diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java index 30d79c6da5a..739fd4c105c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java @@ -1,5 +1,9 @@ package com.bakdata.conquery.models.worker; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.mode.local.SqlEntityResolver; import com.bakdata.conquery.mode.local.SqlStorageHandler; @@ -16,10 +20,6 @@ import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - @Getter @Slf4j public class LocalNamespace extends Namespace { @@ -30,29 +30,35 @@ public class LocalNamespace extends Namespace { private final SqlMatchingStats matchingStats; public LocalNamespace( - DialectBundle dialect, - ObjectMapper preprocessMapper, - NamespaceStorage storage, - ExecutionManager executionManager, - DSLContext dslContext, - SqlStorageHandler storageHandler, - JobManager jobManager, - SearchProcessor filterSearch, - SqlEntityResolver sqlEntityResolver, DatabaseConnectionConfig databaseConfig + DialectBundle dialect, + ObjectMapper preprocessMapper, + NamespaceStorage storage, + ExecutionManager executionManager, + DSLContext dslContext, + SqlStorageHandler storageHandler, + JobManager jobManager, + SearchProcessor filterSearch, + SqlEntityResolver sqlEntityResolver, + DatabaseConnectionConfig databaseConfig ) { super(preprocessMapper, storage, executionManager, jobManager, filterSearch, sqlEntityResolver); this.dslContext = dslContext; this.storageHandler = storageHandler; this.dialect = dialect; - this.matchingStats = new SqlMatchingStats(dslContext, dialect.getFunctionProvider(), databaseConfig.getPrimaryColumn(), databaseConfig.getMatchingStatsWorkers(), databaseConfig.getMatchingStatsRetries()); + this.matchingStats = new SqlMatchingStats( + dslContext, + dialect.getFunctionProvider(), + databaseConfig.getPrimaryColumn(), + databaseConfig.getMatchingStatsWorkers(), + databaseConfig.getMatchingStatsRetries()); } @Override void updateMatchingStats() { getJobManager().addSlowJob( - new UpdateMatchingStatsSqlJob(getStorage().getAllConcepts().toList(), getDataset(), getMatchingStats()) + new UpdateMatchingStatsSqlJob(getStorage().getAllConcepts().toList(), getDataset(), getMatchingStats()) ); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java index dbbefee167b..ec47f337511 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java @@ -61,9 +61,8 @@ public Dataset getDataset() { public void close() { try { filterSearch.stop(); - } - catch (Exception e) { - log.error("Unable to close filter serach of {}",this, e); + } catch (Exception e) { + log.error("Unable to close filter serach of {}", this, e); } try { @@ -100,20 +99,25 @@ public PreviewConfig getPreviewConfig() { } public void updateInternToExternMappings() { - try(Stream> allConcepts = storage.getAllConcepts()) { - allConcepts - .flatMap(c -> c.getConnectors().stream()) - .flatMap(con -> con.getSelects().stream()) - .filter(MappableSingleColumnSelect.class::isInstance) - .map(MappableSingleColumnSelect.class::cast) - .forEach((s) -> jobManager.addSlowJob(new SimpleJob("Update internToExtern Mappings [" + s.getId() + "]", s::loadMapping))); + try (Stream> allConcepts = storage.getAllConcepts()) { + allConcepts.flatMap(c -> c.getConnectors().stream()) + .flatMap(con -> con.getSelects().stream()) + .filter( + MappableSingleColumnSelect.class::isInstance) + .map(MappableSingleColumnSelect.class::cast) + .forEach( + (s) -> jobManager.addSlowJob( + new SimpleJob("Update internToExtern Mappings [" + s.getId() + "]", s::loadMapping))); } - try(Stream secondaryIds = storage.getSecondaryIds()) { - secondaryIds - .filter(desc -> desc.getMapping() != null) - .forEach((s) -> jobManager.addSlowJob(new SimpleJob("Update internToExtern Mappings [" + s.getId() + "]", s.getMapping().resolve()::init))); + try (Stream secondaryIds = storage.getSecondaryIds()) { + secondaryIds.filter(desc -> desc.getMapping() != null) + .forEach( + (s) -> jobManager.addSlowJob( + new SimpleJob( + "Update internToExtern Mappings [" + s.getId() + "]", + s.getMapping().resolve()::init))); } } @@ -121,7 +125,8 @@ public void updateInternToExternMappings() { * Issues a job that initializes the search that is used by the frontend for recommendations in the filter interface of a concept. */ final void updateFilterSearch() { - getJobManager().addSlowJob(filterSearch.createUpdateFilterSearchJob(storage, this::registerColumnValuesInSearch)); + getJobManager().addSlowJob( + filterSearch.createUpdateFilterSearchJob(storage, this::registerColumnValuesInSearch)); } /** @@ -144,14 +149,15 @@ final void updateFilterSearch() { */ public void postprocessData() { - getJobManager().addSlowJob(new SimpleJob( + getJobManager().addSlowJob( + new SimpleJob( "Initiate Update Matching Stats and InternalFilterSearch", () -> { updateInternToExternMappings(); updateMatchingStats(); updateFilterSearch(); } - )); + )); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/ShardNodeInformation.java b/backend/src/main/java/com/bakdata/conquery/models/worker/ShardNodeInformation.java index 295041e379b..8572581f50f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/ShardNodeInformation.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/ShardNodeInformation.java @@ -32,10 +32,11 @@ public ShardNodeInformation(NetworkSession session) { super(session); // This metric tracks when the last message from the corresponding shard was received. - SharedMetricRegistries.getDefault().gauge( + SharedMetricRegistries.getDefault() + .gauge( getLatenessMetricName(), () -> this::getMillisSinceLastStatus - ); + ); } private String getLatenessMetricName() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/ShardWorkers.java b/backend/src/main/java/com/bakdata/conquery/models/worker/ShardWorkers.java index 12a430c4775..e61512da92b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/ShardWorkers.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/ShardWorkers.java @@ -53,9 +53,10 @@ public class ShardWorkers implements NamespacedStorageProvider, Managed { public ShardWorkers( - ThreadPoolDefinition queryThreadPoolDefinition, - InternalMapperFactory internalMapperFactory, - int secondaryIdSubPlanRetention, Clock clock) { + ThreadPoolDefinition queryThreadPoolDefinition, + InternalMapperFactory internalMapperFactory, + int secondaryIdSubPlanRetention, + Clock clock) { this.queryThreadPoolDefinition = queryThreadPoolDefinition; // TODO This shouldn't be coupled to the query thread pool definition @@ -69,14 +70,16 @@ public ShardWorkers( } public Worker openWorker(WorkerStorage storage, boolean failOnError, boolean loadStorage) { - return Worker.create(storage, - this, - queryThreadPoolDefinition, - jobsThreadPool, - clock, failOnError, - secondaryIdSubPlanRetention, - internalMapperFactory.createWorkerPersistenceMapper(storage), - loadStorage + return Worker.create( + storage, + this, + queryThreadPoolDefinition, + jobsThreadPool, + clock, + failOnError, + secondaryIdSubPlanRetention, + internalMapperFactory.createWorkerPersistenceMapper(storage), + loadStorage ); } @@ -87,7 +90,12 @@ public void addWorker(Worker worker) { } @SneakyThrows - public Worker newWorker(Dataset dataset, @NonNull String name, @NonNull NetworkSession session, StoreFactory storageConfig, boolean failOnError) { + public Worker newWorker( + Dataset dataset, + @NonNull String name, + @NonNull NetworkSession session, + StoreFactory storageConfig, + boolean failOnError) { // Create a new worker storage with minimal information and close it immediately so we can pass it to the normal method, that is used to instantiate existing workers try (final WorkerStorage workerStorage = new WorkerStorageImpl(storageConfig, name)) { @@ -106,14 +114,16 @@ public Worker newWorker(Dataset dataset, @NonNull String name, @NonNull NetworkS // WorkerStorageImpl workerStorage = new WorkerStorageImpl(storageConfig, name); final ObjectMapper persistenceMapper = internalMapperFactory.createWorkerPersistenceMapper(workerStorage); - final Worker worker = Worker.create(workerStorage, - this, - queryThreadPoolDefinition, - jobsThreadPool, - clock, failOnError, - secondaryIdSubPlanRetention, - persistenceMapper, - true + final Worker worker = Worker.create( + workerStorage, + this, + queryThreadPoolDefinition, + jobsThreadPool, + clock, + failOnError, + secondaryIdSubPlanRetention, + persistenceMapper, + true ); worker.setSession(session); @@ -142,8 +152,7 @@ removed from dataset2Worker (which is used in deserialization of NamespacedIds, workers.remove(removed.getInfo().getId()); try { removed.remove(); - } - catch (Exception e) { + } catch (Exception e) { log.error("Failed to remove storage {}", removed, e); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java b/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java index 4729b29aa0c..38999858422 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java @@ -57,10 +57,15 @@ public class Worker implements MessageSender.Transforming getAllWorkerIds() { - return getWorkers().stream() - .map(WorkerInformation::getId) - .collect(Collectors.toSet()); + return getWorkers().stream().map(WorkerInformation::getId).collect(Collectors.toSet()); } @@ -72,7 +70,11 @@ public void handleReactionMessage(ReactionMessage message) { final PendingReaction pendingReaction = pendingReactions.get(callerId); if (pendingReaction == null) { - throw new IllegalStateException(String.format("No pending action registered (anymore) for caller id %s from reaction message: %s", callerId, message)); + throw new IllegalStateException( + String.format( + "No pending action registered (anymore) for caller id %s from reaction message: %s", + callerId, + message)); } if (pendingReaction.checkoffWorker(message)) { @@ -134,10 +136,11 @@ public synchronized WorkerInformation getResponsibleWorkerForBucket(int bucket) */ public synchronized WorkerInformation addResponsibility(int bucket) { - final WorkerInformation smallest = workers - .stream() - .min(Comparator.comparing(si -> si.getIncludedBuckets().size())) - .orElseThrow(() -> new IllegalStateException("Unable to find minimum.")); + final WorkerInformation smallest = workers.stream() + .min( + Comparator.comparing(si -> si.getIncludedBuckets().size())) + .orElseThrow( + () -> new IllegalStateException("Unable to find minimum.")); log.debug("Assigning Bucket[{}] to Worker[{}]", bucket, smallest.getId()); @@ -153,15 +156,16 @@ public void register(ShardNodeInformation node, WorkerInformation info) { if (old != null) { old.setIncludedBuckets(info.getIncludedBuckets()); old.setConnectedShardNode(node); - } - else { + } else { info.setConnectedShardNode(node); } addWorker(info); } public synchronized void addWorker(WorkerInformation info) { - Objects.requireNonNull(info.getConnectedShardNode(), () -> String.format("No open connections found for Worker[%s]", info.getId())); + Objects.requireNonNull( + info.getConnectedShardNode(), + () -> String.format("No open connections found for Worker[%s]", info.getId())); workers.add(info); @@ -170,7 +174,8 @@ public synchronized void addWorker(WorkerInformation info) { // This is a completely invalid state from which we should not recover even in production settings. if (old != null && !old.equals(info)) { - throw new IllegalStateException(String.format("Duplicate claims for Bucket[%d] from %s and %s", bucket, old, info)); + throw new IllegalStateException( + String.format("Duplicate claims for Bucket[%d] from %s and %s", bucket, old, info)); } } } @@ -218,10 +223,18 @@ public synchronized boolean checkoffWorker(ReactionMessage message) { } if (!pendingWorkers.remove(workerId)) { - throw new IllegalStateException(String.format("Could not check off worker %s for action-reaction message '%s'. Worker was not checked in.", workerId, parent.getMessageId())); + throw new IllegalStateException( + String.format( + "Could not check off worker %s for action-reaction message '%s'. Worker was not checked in.", + workerId, + parent.getMessageId())); } - log.debug("Checked off worker '{}' for action-reaction message '{}', still waiting for {}.", workerId, parent, pendingWorkers.size()); + log.debug( + "Checked off worker '{}' for action-reaction message '{}', still waiting for {}.", + workerId, + parent, + pendingWorkers.size()); if (!pendingWorkers.isEmpty()) { return false; diff --git a/backend/src/main/java/com/bakdata/conquery/package-info.java b/backend/src/main/java/com/bakdata/conquery/package-info.java index d8e6576d366..6efb5aa773b 100644 --- a/backend/src/main/java/com/bakdata/conquery/package-info.java +++ b/backend/src/main/java/com/bakdata/conquery/package-info.java @@ -1,4 +1,4 @@ /** * The bakdata ConQuery main package containing the main entry point. */ -package com.bakdata.conquery; \ No newline at end of file +package com.bakdata.conquery; diff --git a/backend/src/main/java/com/bakdata/conquery/resources/ResourceConstants.java b/backend/src/main/java/com/bakdata/conquery/resources/ResourceConstants.java index dbee7d929e5..674afdc0efe 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/ResourceConstants.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/ResourceConstants.java @@ -7,15 +7,15 @@ public class ResourceConstants { - public static final String DATASET = "dataset"; + public static final String DATASET = "dataset"; public static final String CONCEPT = "concept"; public static final String CONNECTOR = "connector"; public static final String TABLE = "table"; - public static final String FILTER = "filter"; - public static final String QUERY = "query"; - public static final String FORM_CONFIG = "form-config"; - public static final String FILENAME = "filename"; - public static final String API = "api"; + public static final String FILTER = "filter"; + public static final String QUERY = "query"; + public static final String FORM_CONFIG = "form-config"; + public static final String FILENAME = "filename"; + public static final String API = "api"; public static final String IMPORT_ID = "importId"; public static final String JOB_ID = "jobId"; public static final String OWNER_ID = "ownerId"; @@ -50,13 +50,14 @@ public class ResourceConstants { */ public static TemplateModel getAsTemplateModel() { try { - return new DefaultObjectWrapperBuilder(Freemarker.VERSION) - .build() + return new DefaultObjectWrapperBuilder(Freemarker.VERSION).build() .getStaticModels() - .get(ResourceConstants.class.getName()); - } - catch (TemplateModelException e) { - throw new IllegalStateException("Could not generate template model for " + ResourceConstants.class.getName(), e); + .get( + ResourceConstants.class.getName()); + } catch (TemplateModelException e) { + throw new IllegalStateException( + "Could not generate template model for " + ResourceConstants.class.getName(), + e); } } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/ResourcesProvider.java b/backend/src/main/java/com/bakdata/conquery/resources/ResourcesProvider.java index 81d383981b9..d88fe9cd12e 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/ResourcesProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/ResourcesProvider.java @@ -10,7 +10,8 @@ public interface ResourcesProvider extends Closeable { void registerResources(ManagerNode manager); - + @Override - default void close() throws IOException {} + default void close() throws IOException { + } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminPermissionFilter.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminPermissionFilter.java index 011c155934c..d9213c4ee44 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminPermissionFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminPermissionFilter.java @@ -1,5 +1,8 @@ package com.bakdata.conquery.resources.admin; +import jakarta.ws.rs.NotAuthorizedException; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.container.ContainerRequestFilter; import java.io.IOException; import java.util.Set; @@ -8,9 +11,6 @@ import com.bakdata.conquery.models.auth.permissions.AdminPermission; import com.bakdata.conquery.models.auth.permissions.Authorized; import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; -import jakarta.ws.rs.NotAuthorizedException; -import jakarta.ws.rs.container.ContainerRequestContext; -import jakarta.ws.rs.container.ContainerRequestFilter; /** * Enforces that the requesting subject has the admin permission. diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java index 29e70cdc576..f814b101f57 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java @@ -2,10 +2,10 @@ import static com.bakdata.conquery.resources.ResourceConstants.*; -import java.nio.charset.StandardCharsets; -import java.util.Collections; import jakarta.servlet.ServletRegistration; import jakarta.validation.Validator; +import java.nio.charset.StandardCharsets; +import java.util.Collections; import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.io.freemarker.Freemarker; @@ -78,102 +78,117 @@ public AdminServlet(ManagerNode manager) { final AdminEnvironment admin = manager.getEnvironment().admin(); - ServletRegistration.Dynamic adminServlet = admin.addServlet(ADMIN_SERVLET_PATH, new ServletContainer(jerseyConfig)); + ServletRegistration.Dynamic adminServlet = admin.addServlet( + ADMIN_SERVLET_PATH, + new ServletContainer(jerseyConfig)); adminServlet.addMapping("/" + ADMIN_SERVLET_PATH + "/*"); - admin.addServlet(ADMIN_UI_SERVLET_PATH, new ServletContainer(jerseyConfigUI)).addMapping("/" + ADMIN_UI_SERVLET_PATH + "/*"); + admin.addServlet(ADMIN_UI_SERVLET_PATH, new ServletContainer(jerseyConfigUI)) + .addMapping( + "/" + ADMIN_UI_SERVLET_PATH + "/*"); // Register static asset servlet for admin end - admin.addServlet(ADMIN_ASSETS_PATH, new AssetServlet(ADMIN_ASSETS_PATH, "/" + ADMIN_ASSETS_PATH, null, StandardCharsets.UTF_8)) - .addMapping("/" + ADMIN_ASSETS_PATH + "/*"); + admin.addServlet( + ADMIN_ASSETS_PATH, + new AssetServlet(ADMIN_ASSETS_PATH, "/" + ADMIN_ASSETS_PATH, null, StandardCharsets.UTF_8)) + .addMapping( + "/" + ADMIN_ASSETS_PATH + "/*"); jerseyConfig.register(new JacksonMessageBodyProvider(manager.getEnvironment().getObjectMapper())); adminProcessor = new AdminProcessor( - manager, - manager.getConfig(), - manager.getMetaStorage(), - manager.getDatasetRegistry(), - manager.getJobManager(), - manager.getMaintenanceService(), - manager.getValidator(), - manager.getNodeProvider() + manager, + manager.getConfig(), + manager.getMetaStorage(), + manager.getDatasetRegistry(), + manager.getJobManager(), + manager.getMaintenanceService(), + manager.getValidator(), + manager.getNodeProvider() ); adminDatasetProcessor = new AdminDatasetProcessor( - manager.getConfig(), - manager.getDatasetRegistry(), - manager.getMetaStorage(), - manager.getJobManager(), - manager.getImportHandler(), - manager.getStorageListener(), - manager.getEnvironment() + manager.getConfig(), + manager.getDatasetRegistry(), + manager.getMetaStorage(), + manager.getJobManager(), + manager.getImportHandler(), + manager.getStorageListener(), + manager.getEnvironment() ); jerseyConfig.register(new AbstractBinder() { - @Override - protected void configure() { - bind(manager).to(ManagerNode.class); - bind(manager.getDatasetRegistry()).to(DatasetRegistry.class); - bind(manager.getMetaStorage()).to(MetaStorage.class); - bind(manager.getValidator()).to(Validator.class); - bind(manager.getJobManager()).to(JobManager.class); - bind(manager.getConfig()).to(ConqueryConfig.class); - bind(adminProcessor).to(AdminProcessor.class); - bind(adminDatasetProcessor).to(AdminDatasetProcessor.class); - } - }) - .register(AdminPermissionFilter.class) - .register(IdPathParamConverterProvider.class) - .register(AuthCookieFilter.class) - .register(CsrfTokenCheckFilter.class) - .register(DatasetParamInjector.class); - - - jerseyConfigUI.register(new ViewMessageBodyWriter(manager.getEnvironment().metrics(), Collections.singleton(Freemarker.HTML_RENDERER))) - .register(new AbstractBinder() { - @Override - protected void configure() { - bind(adminProcessor).to(AdminProcessor.class); - bindAsContract(UIProcessor.class); - bind(manager.getDatasetRegistry()).to(DatasetRegistry.class); - bind(manager.getMetaStorage()).to(MetaStorage.class); - bind(manager.getConfig()).to(ConqueryConfig.class); - } - }) - .register(IdPathParamConverterProvider.class) - .register(AdminPermissionFilter.class) - .register(AuthCookieFilter.class) - .register(CsrfTokenSetFilter.class) - .register(DatasetParamInjector.class); + @Override + protected void configure() { + bind(manager).to(ManagerNode.class); + bind(manager.getDatasetRegistry()).to(DatasetRegistry.class); + bind(manager.getMetaStorage()).to(MetaStorage.class); + bind(manager.getValidator()).to(Validator.class); + bind(manager.getJobManager()).to(JobManager.class); + bind(manager.getConfig()).to(ConqueryConfig.class); + bind(adminProcessor).to(AdminProcessor.class); + bind(adminDatasetProcessor).to(AdminDatasetProcessor.class); + } + }) + .register(AdminPermissionFilter.class) + .register(IdPathParamConverterProvider.class) + .register( + AuthCookieFilter.class) + .register(CsrfTokenCheckFilter.class) + .register(DatasetParamInjector.class); + + + jerseyConfigUI.register( + new ViewMessageBodyWriter( + manager.getEnvironment().metrics(), + Collections.singleton(Freemarker.HTML_RENDERER))).register(new AbstractBinder() { + @Override + protected void configure() { + bind(adminProcessor).to(AdminProcessor.class); + bindAsContract(UIProcessor.class); + bind(manager.getDatasetRegistry()).to(DatasetRegistry.class); + bind(manager.getMetaStorage()).to(MetaStorage.class); + bind(manager.getConfig()).to(ConqueryConfig.class); + } + }) + .register(IdPathParamConverterProvider.class) + .register(AdminPermissionFilter.class) + .register( + AuthCookieFilter.class) + .register(CsrfTokenSetFilter.class) + .register(DatasetParamInjector.class); } public void register() { // register root resources - jerseyConfig - .register(AdminDatasetResource.class) - .register(AdminDatasetsResource.class) - .register(AdminConceptsResource.class) - .register(AdminTablesResource.class) - .register(RoleResource.class) - .register(UserResource.class) - .register(GroupResource.class) - .register(PermissionResource.class) - .register(AuthOverviewResource.class) - .register(AdminResource.class) - .register(ConfigApiResource.class); - - jerseyConfigUI - .register(AdminUIResource.class) - .register(RoleUIResource.class) - .register(UserUIResource.class) - .register(GroupUIResource.class) - .register(DatasetsUIResource.class) - .register(TablesUIResource.class) - .register(ConceptsUIResource.class) - .register(ConnectorUIResource.class) - .register(AuthOverviewUIResource.class) - .register(IndexServiceUIResource.class); + jerseyConfig.register(AdminDatasetResource.class) + .register(AdminDatasetsResource.class) + .register( + AdminConceptsResource.class) + .register(AdminTablesResource.class) + .register(RoleResource.class) + .register( + UserResource.class) + .register(GroupResource.class) + .register(PermissionResource.class) + .register( + AuthOverviewResource.class) + .register(AdminResource.class) + .register(ConfigApiResource.class); + + jerseyConfigUI.register(AdminUIResource.class) + .register(RoleUIResource.class) + .register( + UserUIResource.class) + .register(GroupUIResource.class) + .register(DatasetsUIResource.class) + .register( + TablesUIResource.class) + .register(ConceptsUIResource.class) + .register(ConnectorUIResource.class) + .register( + AuthOverviewUIResource.class) + .register(IndexServiceUIResource.class); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ShutdownTask.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ShutdownTask.java index 0965112c782..98cc2b5e224 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ShutdownTask.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ShutdownTask.java @@ -27,7 +27,7 @@ public void serverStarted(Server server) { public void execute(Map> parameters, PrintWriter output) throws Exception { log.info("Received Shutdown command"); - if(server == null) { + if (server == null) { output.print("Server not yet started"); return; } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminConceptsResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminConceptsResource.java index bdbebece420..53c9cf48421 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminConceptsResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminConceptsResource.java @@ -45,4 +45,4 @@ public Concept getConcept() { public void removeConcept() { processor.deleteConcept(concept); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java index f45f5369332..421cda19e42 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java @@ -1,5 +1,12 @@ package com.bakdata.conquery.resources.admin.rest; +import jakarta.inject.Inject; +import jakarta.validation.ConstraintViolation; +import jakarta.ws.rs.BadRequestException; +import jakarta.ws.rs.ForbiddenException; +import jakarta.ws.rs.NotFoundException; +import jakarta.ws.rs.WebApplicationException; +import jakarta.ws.rs.core.Response; import java.io.IOException; import java.io.InputStream; import java.util.Arrays; @@ -11,13 +18,6 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import jakarta.inject.Inject; -import jakarta.validation.ConstraintViolation; -import jakarta.ws.rs.BadRequestException; -import jakarta.ws.rs.ForbiddenException; -import jakarta.ws.rs.NotFoundException; -import jakarta.ws.rs.WebApplicationException; -import jakarta.ws.rs.core.Response; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.io.storage.NamespaceStorage; @@ -102,15 +102,12 @@ public synchronized void deleteDataset(DatasetId dataset) { List
tables = tableStream.toList(); if (!tables.isEmpty()) { throw new WebApplicationException( - String.format( - "Cannot delete dataset `%s`, because it still has tables: `%s`", - dataset, - tables.stream() - .map(Table::getId) - .map(Objects::toString) - .collect(Collectors.joining(",")) - ), - Response.Status.CONFLICT + String.format( + "Cannot delete dataset `%s`, because it still has tables: `%s`", + dataset, + tables.stream().map(Table::getId).map(Objects::toString).collect(Collectors.joining(",")) + ), + Response.Status.CONFLICT ); } } @@ -146,17 +143,23 @@ public synchronized void deleteSecondaryId(SecondaryIdDescriptionId secondaryId) final List dependents; try (Stream
tables = namespace.getStorage().getTables()) { - dependents = tables.map(Table::getColumns).flatMap(Arrays::stream) - .filter(column -> secondaryId.equals(column.getSecondaryId())) - .toList(); + dependents = tables.map(Table::getColumns) + .flatMap(Arrays::stream) + .filter( + column -> secondaryId.equals(column.getSecondaryId())) + .toList(); } if (!dependents.isEmpty()) { - final Set tables = dependents.stream().map(Column::getTable).map(Identifiable::getId).collect(Collectors.toSet()); + final Set tables = dependents.stream() + .map(Column::getTable) + .map(Identifiable::getId) + .collect( + Collectors.toSet()); log.error( - "SecondaryId[{}] still present on {}", - secondaryId, - tables + "SecondaryId[{}] still present on {}", + secondaryId, + tables ); throw new ForbiddenException(String.format("SecondaryId still has dependencies. %s", tables)); @@ -181,12 +184,11 @@ public synchronized void addTable(@NonNull Table table, Namespace namespace) { throw new WebApplicationException("Table already exists", Response.Status.CONFLICT); } - Class mode = - switch (namespace) { - case LocalNamespace ignored -> ValidationMode.Local.class; - case DistributedNamespace ignored -> ValidationMode.Clustered.class; - default -> throw new IllegalStateException("Unexpected Namespace class %s".formatted(namespace.getClass())); - }; + Class mode = switch (namespace) { + case LocalNamespace ignored -> ValidationMode.Local.class; + case DistributedNamespace ignored -> ValidationMode.Clustered.class; + default -> throw new IllegalStateException("Unexpected Namespace class %s".formatted(namespace.getClass())); + }; Set> violations = new HashSet<>(); @@ -229,7 +231,9 @@ public synchronized void addConcept(@NonNull Namespace namespace, @NonNull Conce if (namespaceStorage.hasConcept(concept.getId())) { if (!force) { - throw new WebApplicationException("Can't replace already existing concept " + concept.getId(), Response.Status.CONFLICT); + throw new WebApplicationException( + "Can't replace already existing concept " + concept.getId(), + Response.Status.CONFLICT); } deleteConcept(concept.getId()); log.info("Force deleted previous concept: {}", concept.getId()); @@ -264,20 +268,19 @@ public void setPreviewConfig(PreviewConfig previewConfig, Namespace namespace) { public void setIdMapping(InputStream data, Namespace namespace) { log.info("Received IdMapping for Dataset[{}]", namespace.getDataset().getId()); - final CsvParser parser = config.getCsv() - .withSkipHeader(false) - .withParseHeaders(true) - .createParser(); + final CsvParser parser = config.getCsv().withSkipHeader(false).withParseHeaders(true).createParser(); try { parser.beginParsing(data); - final EntityIdMap mapping = EntityIdMap.generateIdMapping(parser, config.getIdColumns().getIds(), namespace.getStorage()); + final EntityIdMap mapping = EntityIdMap.generateIdMapping( + parser, + config.getIdColumns().getIds(), + namespace.getStorage()); namespace.getStorage().updateIdMapping(mapping); - } - finally { + } finally { parser.stopParsing(); } } @@ -314,9 +317,10 @@ public synchronized List deleteTable(TableId table, boolean force) { final List> dependentConcepts; try (Stream> allConcepts = namespace.getStorage().getAllConcepts()) { dependentConcepts = allConcepts.flatMap(c -> c.getConnectors().stream()) - .filter(con -> con.resolveTableId().equals(table)) - .map(Connector::getConcept) - .collect(Collectors.toList()); + .filter( + con -> con.resolveTableId().equals(table)) + .map(Connector::getConcept) + .collect(Collectors.toList()); } if (force || dependentConcepts.isEmpty()) { @@ -324,9 +328,11 @@ public synchronized List deleteTable(TableId table, boolean force) { deleteConcept(concept.getId()); } - namespace.getStorage().getAllImports() - .filter(imp -> imp.getTable().equals(table)) - .forEach(this::deleteImport); + namespace.getStorage() + .getAllImports() + .filter(imp -> imp.getTable().equals(table)) + .forEach( + this::deleteImport); namespace.getStorage().removeTable(table); storageListener.onRemoveTable(table); @@ -374,16 +380,19 @@ public void addInternToExternMapping(Namespace namespace, InternToExternMapper i public List deleteInternToExternMapping(InternToExternMapperId internToExternMapper, boolean force) { final Namespace namespace = datasetRegistry.get(internToExternMapper.getDataset()); - final Set> dependentConcepts = namespace.getStorage().getAllConcepts() - .filter( - c -> c.getSelects().stream() - .filter(MappableSingleColumnSelect.class::isInstance) + final Set> dependentConcepts = namespace.getStorage() + .getAllConcepts() + .filter( + c -> c.getSelects() + .stream() + .filter(MappableSingleColumnSelect.class::isInstance) - .map(MappableSingleColumnSelect.class::cast) - .map(MappableSingleColumnSelect::getMapping) - .anyMatch(internToExternMapper::equals) - ) - .collect(Collectors.toSet()); + .map(MappableSingleColumnSelect.class::cast) + .map(MappableSingleColumnSelect::getMapping) + .anyMatch( + internToExternMapper::equals) + ) + .collect(Collectors.toSet()); if (force || dependentConcepts.isEmpty()) { for (Concept concept : dependentConcepts) { @@ -416,18 +425,22 @@ public void addSearchIndex(Namespace namespace, SearchIndex searchIndex) { public List deleteSearchIndex(SearchIndexId searchIndex, boolean force) { final Namespace namespace = datasetRegistry.get(searchIndex.getDataset()); - final List> dependentConcepts = namespace.getStorage().getAllConcepts() - .filter( - c -> c.getConnectors().stream() - .map(Connector::getFilters) - .flatMap(Collection::stream) - .filter(SelectFilter.class::isInstance) - .map(SelectFilter.class::cast) - .map(SelectFilter::getTemplate) - .filter(Objects::nonNull) - .anyMatch(searchIndex::equals) - ) - .toList(); + final List> dependentConcepts = namespace.getStorage() + .getAllConcepts() + .filter( + c -> c.getConnectors() + .stream() + .map(Connector::getFilters) + .flatMap(Collection::stream) + .filter( + SelectFilter.class::isInstance) + .map(SelectFilter.class::cast) + .map(SelectFilter::getTemplate) + .filter( + Objects::nonNull) + .anyMatch(searchIndex::equals) + ) + .toList(); if (force || dependentConcepts.isEmpty()) { for (Concept concept : dependentConcepts) { diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetResource.java index 1fc1bc693be..e2989902515 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetResource.java @@ -2,12 +2,6 @@ import static com.bakdata.conquery.resources.ResourceConstants.*; -import java.io.BufferedInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; -import java.util.stream.Collectors; -import java.util.zip.GZIPInputStream; import jakarta.annotation.PostConstruct; import jakarta.inject.Inject; import jakarta.validation.Valid; @@ -25,6 +19,12 @@ import jakarta.ws.rs.Produces; import jakarta.ws.rs.QueryParam; import jakarta.ws.rs.core.MediaType; +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.stream.Collectors; +import java.util.zip.GZIPInputStream; import com.bakdata.conquery.io.jersey.ExtraMimeTypes; import com.bakdata.conquery.models.datasets.Dataset; @@ -174,11 +174,14 @@ public void deleteSecondaryId(@PathParam(SECONDARY_ID) SecondaryIdDescriptionId @DELETE @Path("searchIndex/{" + SEARCH_INDEX_ID + "}") - public List deleteSearchIndex(@PathParam(SEARCH_INDEX_ID) SearchIndexId searchIndex, @QueryParam("force") @DefaultValue("false") boolean force) { + public List deleteSearchIndex( + @PathParam(SEARCH_INDEX_ID) SearchIndexId searchIndex, + @QueryParam("force") @DefaultValue("false") boolean force) { final List conceptIds = processor.deleteSearchIndex(searchIndex, force); if (!conceptIds.isEmpty() && !force) { - throw new BadRequestException(String.format("Cannot delete search index because it is used by these concepts: %s", conceptIds)); + throw new BadRequestException( + String.format("Cannot delete search index because it is used by these concepts: %s", conceptIds)); } return conceptIds; } @@ -186,8 +189,8 @@ public List deleteSearchIndex(@PathParam(SEARCH_INDEX_ID) SearchIndex @DELETE @Path("internToExtern/{" + INTERN_TO_EXTERN_ID + "}") public List deleteInternToExternMapping( - @PathParam(INTERN_TO_EXTERN_ID) InternToExternMapperId internToExternMapper, - @QueryParam("force") @DefaultValue("false") boolean force) { + @PathParam(INTERN_TO_EXTERN_ID) InternToExternMapperId internToExternMapper, + @QueryParam("force") @DefaultValue("false") boolean force) { return processor.deleteInternToExternMapping(internToExternMapper, force); } @@ -212,10 +215,7 @@ public List listTables() { @GET @Path("concepts") public List listConcepts() { - return namespace.getStorage() - .getAllConcepts() - .map(Concept::getId) - .collect(Collectors.toList()); + return namespace.getStorage().getAllConcepts().map(Concept::getId).collect(Collectors.toList()); } @DELETE diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetsResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetsResource.java index 904c0c36a96..947e3a6d575 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetsResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetsResource.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.resources.admin.rest; -import java.util.stream.Stream; import jakarta.inject.Inject; import jakarta.validation.Valid; import jakarta.validation.constraints.NotNull; @@ -9,6 +8,7 @@ import jakarta.ws.rs.POST; import jakarta.ws.rs.Path; import jakarta.ws.rs.Produces; +import java.util.stream.Stream; import com.bakdata.conquery.io.jersey.ExtraMimeTypes; import com.bakdata.conquery.models.datasets.Dataset; diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminProcessor.java index ff150c6555a..22cfe4841a2 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminProcessor.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.resources.admin.rest; +import jakarta.validation.Validator; import java.io.StringWriter; import java.util.ArrayList; import java.util.Collection; @@ -12,7 +13,6 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; -import jakarta.validation.Validator; import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.io.jackson.Jackson; @@ -132,8 +132,7 @@ public void addUsers(List users) { for (User user : users) { try { addUser(user); - } - catch (Exception e) { + } catch (Exception e) { log.error("Failed to add User: {}", user, e); } } @@ -152,8 +151,7 @@ public void addGroups(List groups) { for (Group group : groups) { try { addGroup(group); - } - catch (Exception e) { + } catch (Exception e) { log.error("Failed to add Group: {}", group, e); } } @@ -218,14 +216,11 @@ public String getPermissionOverviewAsCSV() { public String getPermissionOverviewAsCSV(Stream users) { final StringWriter sWriter = new StringWriter(); final CsvWriter writer = config.getCsv().createWriter(sWriter); - final List scope = config - .getAuthorizationRealms() - .getOverviewScope(); + final List scope = config.getAuthorizationRealms().getOverviewScope(); // Header writeAuthOverviewHeader(writer, scope); // Body - users.forEach(user -> - writeAuthOverviewUser(writer, scope, user, storage, config) + users.forEach(user -> writeAuthOverviewUser(writer, scope, user, storage, config) ); return sWriter.toString(); } @@ -243,16 +238,27 @@ private static void writeAuthOverviewHeader(CsvWriter writer, List scope /** * Writes the given {@link User}s (one per line) with their effective permission to the specified CSV writer. */ - private static void writeAuthOverviewUser(CsvWriter writer, List scope, User user, MetaStorage storage, ConqueryConfig config) { + private static void writeAuthOverviewUser( + CsvWriter writer, + List scope, + User user, + MetaStorage storage, + ConqueryConfig config) { // Print the user in the first column writer.addValue(String.format("%s %s", user.getLabel(), ConqueryEscape.unescape(user.getName()))); // Print the permission per domain in the remaining columns - final Multimap permissions = AuthorizationHelper.getEffectiveUserPermissions(user, scope, storage); + final Multimap permissions = AuthorizationHelper.getEffectiveUserPermissions( + user, + scope, + storage); for (String domain : scope) { - writer.addValue(permissions.get(domain).stream() - .map(Object::toString) - .collect(Collectors.joining(config.getCsv().getLineSeparator()))); + writer.addValue( + permissions.get(domain) + .stream() + .map(Object::toString) + .collect( + Collectors.joining(config.getCsv().getLineSeparator()))); } writer.writeValuesToRow(); } @@ -267,26 +273,23 @@ public String getPermissionOverviewAsCSV(GroupId groupId) { public boolean isBusy() { //Note that this does not and cannot check for fast jobs! - return getJobs().stream() - .map(JobManagerStatus::getJobs) - .anyMatch(Predicate.not(Collection::isEmpty)); + return getJobs().stream().map(JobManagerStatus::getJobs).anyMatch(Predicate.not(Collection::isEmpty)); } public Collection getJobs() { final List out = new ArrayList<>(); - out.add(JobManagerStatus.builder() - .origin("Manager") - .jobs(getJobManager().getJobStatus()) - .build()); + out.add(JobManagerStatus.builder().origin("Manager").jobs(getJobManager().getJobStatus()).build()); for (Namespace namespace : getDatasetRegistry().getNamespaces()) { - out.add(JobManagerStatus.builder() - .origin("Manager") - .dataset(namespace.getDataset().getId()) - .jobs(namespace.getJobManager().getJobStatus()) - .build()); + out.add( + JobManagerStatus.builder() + .origin("Manager") + .dataset(namespace.getDataset().getId()) + .jobs( + namespace.getJobManager().getJobStatus()) + .build()); } for (ShardNodeInformation si : nodeProvider.get()) { @@ -309,8 +312,7 @@ public Object executeScript(String script) { try { return groovy.evaluate(script); - } - catch (Exception e) { + } catch (Exception e) { return ExceptionUtils.getStackTrace(e); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java index 50600bb548e..93414fc60ff 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java @@ -3,13 +3,6 @@ import static com.bakdata.conquery.resources.ResourceConstants.INDEX_SERVICE_PATH_ELEMENT; import static com.bakdata.conquery.resources.ResourceConstants.JOB_ID; -import java.time.LocalDate; -import java.util.Collection; -import java.util.Objects; -import java.util.Optional; -import java.util.OptionalLong; -import java.util.UUID; -import java.util.stream.Stream; import jakarta.inject.Inject; import jakarta.ws.rs.Consumes; import jakarta.ws.rs.GET; @@ -21,6 +14,13 @@ import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.UriBuilder; +import java.time.LocalDate; +import java.util.Collection; +import java.util.Objects; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.UUID; +import java.util.stream.Stream; import com.bakdata.conquery.apiv1.execution.ExecutionStatus; import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; @@ -97,7 +97,10 @@ public boolean isBusy() { @GET @Path("/queries") - public Stream getQueries(@Auth Subject currentUser, @QueryParam("limit") OptionalLong maybeLimit, @QueryParam("since") Optional maybeSince) { + public Stream getQueries( + @Auth Subject currentUser, + @QueryParam("limit") OptionalLong maybeLimit, + @QueryParam("since") Optional maybeSince) { final LocalDate since = maybeSince.map(LocalDate::parse).orElse(LocalDate.now()); final long limit = maybeLimit.orElse(100); @@ -106,26 +109,29 @@ public Stream getQueries(@Auth Subject currentUser, @QueryParam return storage.getAllExecutions() - .filter(t -> t.getCreationTime().toLocalDate().isAfter(since) || t.getCreationTime().toLocalDate().isEqual(since)) - .limit(limit) - .map(t -> { - try { - if (t.isInitialized()) { - final Namespace namespace = processor.getDatasetRegistry().get(t.getDataset()); - return t.buildStatusFull(currentUser, namespace); - } - - return t.buildStatusOverview(currentUser); - } - catch (ConqueryError e) { - // Initialization of execution probably failed, so we construct a status based on the overview status - final FullExecutionStatus fullExecutionStatus = new FullExecutionStatus(); - t.setStatusBase(currentUser, fullExecutionStatus); - fullExecutionStatus.setStatus(ExecutionState.FAILED); - fullExecutionStatus.setError(e); - return fullExecutionStatus; - } - }); + .filter( + t -> t.getCreationTime().toLocalDate().isAfter(since) || t.getCreationTime() + .toLocalDate() + .isEqual( + since)) + .limit(limit) + .map(t -> { + try { + if (t.isInitialized()) { + final Namespace namespace = processor.getDatasetRegistry().get(t.getDataset()); + return t.buildStatusFull(currentUser, namespace); + } + + return t.buildStatusOverview(currentUser); + } catch (ConqueryError e) { + // Initialization of execution probably failed, so we construct a status based on the overview status + final FullExecutionStatus fullExecutionStatus = new FullExecutionStatus(); + t.setStatusBase(currentUser, fullExecutionStatus); + fullExecutionStatus.setStatus(ExecutionState.FAILED); + fullExecutionStatus.setError(e); + return fullExecutionStatus; + } + }); } @POST diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminTablesResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminTablesResource.java index 892036c1b70..aced0e85378 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminTablesResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminTablesResource.java @@ -2,8 +2,6 @@ import static com.bakdata.conquery.resources.ResourceConstants.*; -import java.util.List; -import java.util.stream.Collectors; import jakarta.inject.Inject; import jakarta.ws.rs.Consumes; import jakarta.ws.rs.DELETE; @@ -15,6 +13,8 @@ import jakarta.ws.rs.QueryParam; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.Response.Status; +import java.util.List; +import java.util.stream.Collectors; import com.bakdata.conquery.apiv1.AdditionalMediaTypes; import com.bakdata.conquery.io.jersey.ExtraMimeTypes; @@ -72,7 +72,11 @@ public Response remove(@PathParam(TABLE) TableId table, @QueryParam("force") @De public List listImports(@PathParam(TABLE) TableId table) { Namespace namespace = processor.getDatasetRegistry().get(dataset); - return namespace.getStorage().getAllImports().filter(imp -> imp.getTable().equals(table)).collect(Collectors.toList()); + return namespace.getStorage() + .getAllImports() + .filter(imp -> imp.getTable().equals(table)) + .collect( + Collectors.toList()); } @DELETE @@ -89,4 +93,4 @@ public Import getImport(@PathParam(TABLE) TableId table, @PathParam(IMPORT_ID) I } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AuthOverviewResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AuthOverviewResource.java index 6982068d833..eaa7e1bae40 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AuthOverviewResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AuthOverviewResource.java @@ -28,18 +28,22 @@ public class AuthOverviewResource { @GET @Path("csv") public Response getPermissionOverviewAsCSV() { - return Response - .ok(processor.getPermissionOverviewAsCSV()) - .header("Content-Disposition", "attachment; filename=\"authOverview.csv\"") - .build(); + return Response.ok(processor.getPermissionOverviewAsCSV()) + .header( + "Content-Disposition", + "attachment; filename=\"authOverview.csv\"") + .build(); } @GET @Path("csv/group/{" + GROUP_ID + "}") public Response getPermissionOverviewAsCSV(@PathParam(GROUP_ID) GroupId group) { - return Response - .ok(processor.getPermissionOverviewAsCSV(group)) - .header("Content-Disposition", String.format("attachment; filename=\"authOverview_%s.csv\"", FileUtil.makeSafeFileName(group.getGroup()))) - .build(); + return Response.ok(processor.getPermissionOverviewAsCSV(group)) + .header( + "Content-Disposition", + String.format( + "attachment; filename=\"authOverview_%s.csv\"", + FileUtil.makeSafeFileName(group.getGroup()))) + .build(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/ConfigApiResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/ConfigApiResource.java index cf5187d96d4..8a2df5001b4 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/ConfigApiResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/ConfigApiResource.java @@ -1,8 +1,8 @@ package com.bakdata.conquery.resources.admin.rest; +import jakarta.inject.Inject; import java.util.Iterator; import java.util.List; -import jakarta.inject.Inject; import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.models.api.openapi.ConfigApi; @@ -22,7 +22,7 @@ public String deleteFormBackend(String configId) { Iterator iterator = config.getPlugins().iterator(); while (iterator.hasNext()) { PluginConfig next = iterator.next(); - if (next instanceof FormBackendConfig backendConfig && backendConfig.getId().equals(configId)){ + if (next instanceof FormBackendConfig backendConfig && backendConfig.getId().equals(configId)) { // Remove from Plugins iterator.remove(); diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/GroupResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/GroupResource.java index 5e73d17f22c..cbacd81a9f0 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/GroupResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/GroupResource.java @@ -2,8 +2,6 @@ import static com.bakdata.conquery.resources.ResourceConstants.*; -import java.util.Collection; -import java.util.List; import jakarta.inject.Inject; import jakarta.validation.constraints.NotEmpty; import jakarta.ws.rs.Consumes; @@ -14,6 +12,8 @@ import jakarta.ws.rs.PathParam; import jakarta.ws.rs.Produces; import jakarta.ws.rs.core.MediaType; +import java.util.Collection; +import java.util.List; import com.bakdata.conquery.models.auth.entities.Group; import com.bakdata.conquery.models.identifiable.ids.specific.GroupId; diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/PermissionResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/PermissionResource.java index b8f2498d55e..5e93ebd2f5a 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/PermissionResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/PermissionResource.java @@ -1,14 +1,15 @@ package com.bakdata.conquery.resources.admin.rest; -import com.bakdata.conquery.models.auth.permissions.WildcardPermission; -import com.bakdata.conquery.models.identifiable.ids.specific.PermissionOwnerId; -import com.bakdata.conquery.util.validation.ValidConqueryPermission; +import static com.bakdata.conquery.resources.ResourceConstants.OWNER_ID; + import jakarta.inject.Inject; import jakarta.ws.rs.*; import jakarta.ws.rs.core.MediaType; -import lombok.RequiredArgsConstructor; -import static com.bakdata.conquery.resources.ResourceConstants.OWNER_ID; +import com.bakdata.conquery.models.auth.permissions.WildcardPermission; +import com.bakdata.conquery.models.identifiable.ids.specific.PermissionOwnerId; +import com.bakdata.conquery.util.validation.ValidConqueryPermission; +import lombok.RequiredArgsConstructor; @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) @@ -31,7 +32,7 @@ public class PermissionResource { public void createPermission(@ValidConqueryPermission String permission) { processor.createPermission(owner, new WildcardPermission(permission)); } - + @DELETE public void deletePermission(@ValidConqueryPermission String permission) { processor.deletePermission(owner, new WildcardPermission(permission)); diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/RoleResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/RoleResource.java index 5a0d1d2586a..61f8a5deb7d 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/RoleResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/RoleResource.java @@ -3,7 +3,6 @@ import static com.bakdata.conquery.resources.ResourceConstants.ROLES_PATH_ELEMENT; import static com.bakdata.conquery.resources.ResourceConstants.ROLE_ID; -import java.util.Collection; import jakarta.inject.Inject; import jakarta.ws.rs.Consumes; import jakarta.ws.rs.DELETE; @@ -13,6 +12,7 @@ import jakarta.ws.rs.PathParam; import jakarta.ws.rs.Produces; import jakarta.ws.rs.core.MediaType; +import java.util.Collection; import com.bakdata.conquery.models.auth.entities.Role; import com.bakdata.conquery.models.exceptions.JSONException; diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UIProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UIProcessor.java index e9263f4a065..e4351e81889 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UIProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UIProcessor.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.resources.admin.rest; +import jakarta.inject.Inject; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -13,7 +14,6 @@ import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; -import jakarta.inject.Inject; import com.bakdata.conquery.io.cps.CPSTypeIdResolver; import com.bakdata.conquery.io.storage.MetaStorage; @@ -80,17 +80,28 @@ public FrontendAuthOverview getAuthOverview() { Collection overview = new TreeSet<>(); for (User user : getStorage().getAllUsers().toList()) { Collection userGroups = AuthorizationHelper.getGroupsOf(user, getStorage()); - Set effectiveRoles = user.getRoles().stream() - .map(getStorage()::getRole) - // Filter role_ids that might not map TODO how do we handle those - .filter(Predicate.not(Objects::isNull)) - .collect(Collectors.toCollection(HashSet::new)); - userGroups.forEach(g -> effectiveRoles.addAll(g.getRoles().stream() - .map(getStorage()::getRole) - // Filter role_ids that might not map TODO how do we handle those - .filter(Predicate.not(Objects::isNull)) - .sorted().toList())); - overview.add(FrontendAuthOverview.OverviewRow.builder().user(user).groups(userGroups).effectiveRoles(effectiveRoles).build()); + Set effectiveRoles = user.getRoles() + .stream() + .map(getStorage()::getRole) + // Filter role_ids that might not map TODO how do we handle those + .filter(Predicate.not(Objects::isNull)) + .collect(Collectors.toCollection(HashSet::new)); + userGroups.forEach( + g -> effectiveRoles.addAll( + g.getRoles() + .stream() + .map(getStorage()::getRole) + // Filter role_ids that might not map TODO how do we handle those + .filter(Predicate.not(Objects::isNull)) + .sorted() + .toList())); + overview.add( + FrontendAuthOverview.OverviewRow.builder() + .user(user) + .groups(userGroups) + .effectiveRoles( + effectiveRoles) + .build()); } return FrontendAuthOverview.builder().overview(overview).build(); } @@ -105,32 +116,31 @@ public FrontendGroupContent getGroupContent(Group group) { ArrayList availableMembers = new ArrayList<>(getStorage().getAllUsers().toList()); availableMembers.removeIf(u -> membersIds.contains(u.getId())); - List members = membersIds.stream() - .map(id -> { - User user = id.get(); - if (user != null) { - return getUserContent(user); - } - return FrontendUserContent.builder().id(id).build(); - }) - .toList(); - - List roles = group.getRoles().stream() - .map(this::getFrontendRoleContent) - .toList(); - - return FrontendGroupContent - .builder() - .resolvable(true) - .label(group.getLabel()) - .id(group.getId()) - .members(members) - .availableMembers(availableMembers) - .roles(roles) - .availableRoles(getStorage().getAllRoles().toList()) - .permissions(wrapInFEPermission(group.getPermissions())) - .permissionTemplateMap(preparePermissionTemplate()) - .build(); + List members = membersIds.stream().map(id -> { + User user = id.get(); + if (user != null) { + return getUserContent(user); + } + return FrontendUserContent.builder().id(id).build(); + }).toList(); + + List roles = group.getRoles().stream().map(this::getFrontendRoleContent).toList(); + + return FrontendGroupContent.builder() + .resolvable(true) + .label(group.getLabel()) + .id(group.getId()) + .members( + members) + .availableMembers(availableMembers) + .roles(roles) + .availableRoles( + getStorage().getAllRoles().toList()) + .permissions( + wrapInFEPermission(group.getPermissions())) + .permissionTemplateMap( + preparePermissionTemplate()) + .build(); } private FrontendRoleContent getFrontendRoleContent(RoleId id) { @@ -154,32 +164,38 @@ public FrontendUserContent getUserContent(User user) { final Collection availableGroups = new ArrayList<>(getStorage().getAllGroups().toList()); availableGroups.removeIf(g -> g.containsUser(user.getId())); - return FrontendUserContent - .builder() - .resolvable(true) - .label(user.getLabel()) - .id(user.getId()) - .groups(AuthorizationHelper.getGroupsOf(user, getStorage())) - .availableGroups(availableGroups) - .roles(user.getRoles().stream().map(this::getFrontendRoleContent).collect(Collectors.toList())) - .availableRoles(getStorage().getAllRoles().toList()) - .permissions(wrapInFEPermission(user.getPermissions())) - .permissionTemplateMap(preparePermissionTemplate()) - .build(); + return FrontendUserContent.builder() + .resolvable(true) + .label(user.getLabel()) + .id(user.getId()) + .groups( + AuthorizationHelper.getGroupsOf(user, getStorage())) + .availableGroups(availableGroups) + .roles( + user.getRoles().stream().map(this::getFrontendRoleContent).collect(Collectors.toList())) + .availableRoles( + getStorage().getAllRoles().toList()) + .permissions( + wrapInFEPermission(user.getPermissions())) + .permissionTemplateMap( + preparePermissionTemplate()) + .build(); } public FrontendRoleContent getRoleContent(RoleId roleId) { final Role role = getStorage().getRole(roleId); return FrontendRoleContent.builder() - .resolvable(true) - .permissions(wrapInFEPermission(role.getPermissions())) - .permissionTemplateMap(preparePermissionTemplate()) - .users(getUsers(role)) - .groups(getGroups(role)) - .id(role.getId()) - .label(role.getLabel()) - .build(); + .resolvable(true) + .permissions( + wrapInFEPermission(role.getPermissions())) + .permissionTemplateMap(preparePermissionTemplate()) + .users( + getUsers(role)) + .groups(getGroups(role)) + .id(role.getId()) + .label(role.getLabel()) + .build(); } private SortedSet wrapInFEPermission(Collection permissions) { @@ -195,16 +211,17 @@ private Map, List>> preparePermissionTemplate( Map, List>> permissionTemplateMap = new HashMap<>(); // Grab all possible permission types for the "Create Permission" section - Set> permissionTypes = CPSTypeIdResolver - .listImplementations(StringPermissionBuilder.class); + Set> permissionTypes = CPSTypeIdResolver.listImplementations( + StringPermissionBuilder.class); for (Class permissionType : permissionTypes) { try { - StringPermissionBuilder instance = (StringPermissionBuilder) permissionType.getField("INSTANCE").get(null); + StringPermissionBuilder instance = (StringPermissionBuilder) permissionType.getField("INSTANCE") + .get( + null); // Right argument is for possible targets of a specific permission type, but it // is left empty for now. permissionTemplateMap.put(instance.getDomain(), Pair.of(instance.getAllowedAbilities(), List.of())); - } - catch (IllegalArgumentException | IllegalAccessException | NoSuchFieldException | SecurityException e) { + } catch (IllegalArgumentException | IllegalAccessException | NoSuchFieldException | SecurityException e) { log.error("Could not access allowed abilities for permission type: {}", permissionType, e); } @@ -213,14 +230,19 @@ private Map, List>> preparePermissionTemplate( } public List getUsers(Role role) { - return getStorage().getAllUsers().filter(u -> u.getRoles().contains(role.getId())).sorted().collect(Collectors.toList()); + return getStorage().getAllUsers() + .filter(u -> u.getRoles().contains(role.getId())) + .sorted() + .collect( + Collectors.toList()); } private List getGroups(Role role) { return getStorage().getAllGroups() - .filter(g -> g.getRoles().contains(role.getId())) - .sorted() - .collect(Collectors.toList()); + .filter(g -> g.getRoles().contains(role.getId())) + .sorted() + .collect( + Collectors.toList()); } public TableStatistics getTableStatistics(TableId tableId) { @@ -231,22 +253,20 @@ public TableStatistics getTableStatistics(TableId tableId) { final long entries = imports.stream().mapToLong(Import::getNumberOfEntries).sum(); return new TableStatistics( - table, - entries, - //total size of entries - imports.stream() - .mapToLong(Import::estimateMemoryConsumption) - .sum(), - // Total size of CBlocks - imports.stream() - .mapToLong(imp -> calculateCBlocksSizeBytes(imp, storage.getAllConcepts())) - .sum(), - imports, - storage.getAllConcepts() - .map(Concept::getConnectors) - .flatMap(Collection::stream) - .filter(conn -> conn.resolveTableId().equals(tableId)) - .map(Connector::getConcept).collect(Collectors.toSet()) + table, + entries, + //total size of entries + imports.stream().mapToLong(Import::estimateMemoryConsumption).sum(), + // Total size of CBlocks + imports.stream().mapToLong(imp -> calculateCBlocksSizeBytes(imp, storage.getAllConcepts())).sum(), + imports, + storage.getAllConcepts() + .map(Concept::getConnectors) + .flatMap(Collection::stream) + .filter( + conn -> conn.resolveTableId().equals(tableId)) + .map(Connector::getConcept) + .collect(Collectors.toSet()) ); } @@ -259,21 +279,23 @@ public static long calculateCBlocksSizeBytes(Import imp, Stream ((TreeConcept) concept).getConnectors().stream()) - .filter(con -> con.resolveTableId().equals(imp.getTable())) - .mapToLong(con -> { - // Per event an int array is stored marking the path to the concept child. - final double avgDepth = con.getConcept() - .getAllChildren() - .mapToInt(ConceptTreeChild::getDepth) - .average() - .orElse(1d); - - return CBlock.estimateMemoryBytes(imp.getNumberOfEntities(), imp.getNumberOfEntries(), avgDepth); - }) - .sum(); + return concepts.filter(TreeConcept.class::isInstance) + .flatMap( + concept -> ((TreeConcept) concept).getConnectors().stream()) + .filter( + con -> con.resolveTableId().equals(imp.getTable())) + .mapToLong(con -> { + // Per event an int array is stored marking the path to the concept child. + final double avgDepth = con.getConcept() + .getAllChildren() + .mapToInt( + ConceptTreeChild::getDepth) + .average() + .orElse(1d); + + return CBlock.estimateMemoryBytes(imp.getNumberOfEntities(), imp.getNumberOfEntries(), avgDepth); + }) + .sum(); } public ImportStatistics getImportStatistics(ImportId importId) { diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UserAuthenticationManagementResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UserAuthenticationManagementResource.java index 68c034889ff..3afdfe5de59 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UserAuthenticationManagementResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UserAuthenticationManagementResource.java @@ -38,8 +38,8 @@ public Response addUser(ProtoUser pUser) { @PUT @Consumes(MediaType.APPLICATION_JSON) public Response updateUser(ProtoUser pUser) { - - if(processor.updateUser(pUser)) { + + if (processor.updateUser(pUser)) { return Response.status(Status.CREATED).build(); } return Response.serverError().status(Status.CONFLICT).build(); diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UserResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UserResource.java index cd14d36a11e..06dfcc92725 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UserResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UserResource.java @@ -2,8 +2,6 @@ import static com.bakdata.conquery.resources.ResourceConstants.*; -import java.util.Collection; -import java.util.List; import jakarta.inject.Inject; import jakarta.validation.Valid; import jakarta.validation.constraints.NotEmpty; @@ -14,6 +12,8 @@ import jakarta.ws.rs.PathParam; import jakarta.ws.rs.Produces; import jakarta.ws.rs.core.MediaType; +import java.util.Collection; +import java.util.List; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.identifiable.ids.specific.RoleId; diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/AdminUIResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/AdminUIResource.java index f6b9854462c..723cb9807ba 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/AdminUIResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/AdminUIResource.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.resources.admin.ui; -import java.net.URI; -import java.util.Objects; import jakarta.inject.Inject; import jakarta.ws.rs.GET; import jakarta.ws.rs.Path; @@ -11,6 +9,8 @@ import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.NewCookie; import jakarta.ws.rs.core.Response; +import java.net.URI; +import java.util.Objects; import com.bakdata.conquery.models.auth.entities.Subject; import com.bakdata.conquery.models.auth.web.csrf.CsrfTokenSetFilter; @@ -24,34 +24,43 @@ @Produces(MediaType.TEXT_HTML) @Path("/") -@RequiredArgsConstructor(onConstructor_=@Inject) +@RequiredArgsConstructor(onConstructor_ = @Inject) public class AdminUIResource { private final UIProcessor uiProcessor; @Context private ContainerRequestContext requestContext; + @GET public View getIndex() { - return new UIView<>("index.html.ftl", uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext))); + return new UIView<>( + "index.html.ftl", + uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext))); } @GET @Path("script") public View getScript() { - return new UIView<>("script.html.ftl", uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext))); + return new UIView<>( + "script.html.ftl", + uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext))); } @GET @Path("jobs") public View getJobs() { - return new UIView<>("jobs.html.ftl", uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), uiProcessor.getAdminProcessor() - .getJobs()); + return new UIView<>( + "jobs.html.ftl", + uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), + uiProcessor.getAdminProcessor().getJobs()); } @GET @Path("queries") public View getQueries() { - return new UIView<>("queries.html.ftl", uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext))); + return new UIView<>( + "queries.html.ftl", + uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext))); } @@ -59,11 +68,17 @@ public View getQueries() { @Path("logout") public Response logout(@Auth Subject user) { // Invalidate all cookies. At the moment the adminEnd uses cookies only for authentication, so this does not interfere with other things - final NewCookie[] expiredCookies = requestContext.getCookies().keySet().stream().map(AuthenticationConfig::expireCookie).toArray(NewCookie[]::new); + final NewCookie[] expiredCookies = requestContext.getCookies() + .keySet() + .stream() + .map( + AuthenticationConfig::expireCookie) + .toArray(NewCookie[]::new); final URI logout = user.getAuthenticationInfo().getFrontChannelLogout(); - return Response.seeOther(Objects.requireNonNullElseGet(logout, () -> URI.create("/" + ResourceConstants.ADMIN_UI_SERVLET_PATH))) - .cookie(expiredCookies) - .build(); + return Response.seeOther( + Objects.requireNonNullElseGet( + logout, + () -> URI.create("/" + ResourceConstants.ADMIN_UI_SERVLET_PATH))).cookie(expiredCookies).build(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/AuthOverviewUIResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/AuthOverviewUIResource.java index 631ef8878af..06b8105c784 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/AuthOverviewUIResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/AuthOverviewUIResource.java @@ -24,7 +24,10 @@ public class AuthOverviewUIResource { @GET public View getOverview(@Context ContainerRequestContext request) { - return new UIView<>("authOverview.html.ftl", uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(request)), uiProcessor.getAuthOverview()); + return new UIView<>( + "authOverview.html.ftl", + uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(request)), + uiProcessor.getAuthOverview()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/ConceptsUIResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/ConceptsUIResource.java index e21d44ebc58..c0ccf7aedad 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/ConceptsUIResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/ConceptsUIResource.java @@ -45,9 +45,9 @@ public class ConceptsUIResource { @GET public View getConceptView() { return new UIView<>( - "concept.html.ftl", - uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(request)), - concept.resolve() + "concept.html.ftl", + uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(request)), + concept.resolve() ); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/DatasetsUIResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/DatasetsUIResource.java index a03a7f52069..c3d81f3ead5 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/DatasetsUIResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/DatasetsUIResource.java @@ -3,8 +3,6 @@ import static com.bakdata.conquery.resources.ResourceConstants.DATASET; import static com.bakdata.conquery.resources.admin.rest.UIProcessor.calculateCBlocksSizeBytes; -import java.util.Collection; -import java.util.stream.Collectors; import jakarta.inject.Inject; import jakarta.ws.rs.GET; import jakarta.ws.rs.Path; @@ -13,6 +11,8 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.MediaType; +import java.util.Collection; +import java.util.stream.Collectors; import com.bakdata.conquery.models.auth.web.csrf.CsrfTokenSetFilter; import com.bakdata.conquery.models.datasets.Dataset; @@ -59,9 +59,9 @@ public class DatasetsUIResource { @Produces(MediaType.TEXT_HTML) public View listDatasetsUI() { return new UIView<>( - "datasets.html.ftl", - uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), - uiProcessor.getDatasetRegistry().getAllDatasets().map(DatasetId::resolve).toList() + "datasets.html.ftl", + uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), + uiProcessor.getDatasetRegistry().getAllDatasets().map(DatasetId::resolve).toList() ); } @@ -71,37 +71,51 @@ public View listDatasetsUI() { public View getDataset(@PathParam(DATASET) DatasetId dataset) { final Namespace namespace = uiProcessor.getDatasetRegistry().get(dataset); return new UIView<>( - "dataset.html.ftl", - uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), - new DatasetInfos( - namespace.getDataset(), - namespace.getStorage().getSecondaryIds().toList(), - namespace.getStorage().getInternToExternMappers().toList(), - namespace.getStorage().getSearchIndices().toList(), - namespace.getStorage().getTables() - .map(table -> new TableInfos( - table.getId(), - table.getName(), - table.getLabel(), - StringUtils.abbreviate(table.findImports(namespace.getStorage()) - .map(Import::getName) - .sorted() - .collect(Collectors.joining(", ")), ABBREVIATION_MARKER, MAX_IMPORTS_TEXT_LENGTH), - table.findImports(namespace.getStorage()).mapToLong(Import::getNumberOfEntries).sum() - )) - .collect(Collectors.toList()), - namespace.getStorage().getAllConcepts().toList(), - // Total size of CBlocks - namespace - .getStorage().getTables() - .flatMap(table -> table.findImports(namespace.getStorage())) - .mapToLong(imp -> calculateCBlocksSizeBytes( - imp, namespace.getStorage().getAllConcepts() - )) - .sum(), - // total size of entries - namespace.getStorage().getAllImports().map(Id::resolve).mapToLong(Import::estimateMemoryConsumption).sum() - ) + "dataset.html.ftl", + uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), + new DatasetInfos( + namespace.getDataset(), + namespace.getStorage().getSecondaryIds().toList(), + namespace.getStorage().getInternToExternMappers().toList(), + namespace.getStorage().getSearchIndices().toList(), + namespace.getStorage() + .getTables() + .map( + table -> new TableInfos( + table.getId(), + table.getName(), + table.getLabel(), + StringUtils.abbreviate( + table.findImports(namespace.getStorage()) + .map(Import::getName) + .sorted() + .collect( + Collectors.joining(", ")), + ABBREVIATION_MARKER, + MAX_IMPORTS_TEXT_LENGTH), + table.findImports(namespace.getStorage()).mapToLong(Import::getNumberOfEntries).sum() + )) + .collect(Collectors.toList()), + namespace.getStorage().getAllConcepts().toList(), + // Total size of CBlocks + namespace.getStorage() + .getTables() + .flatMap( + table -> table.findImports(namespace.getStorage())) + .mapToLong( + imp -> calculateCBlocksSizeBytes( + imp, + namespace.getStorage().getAllConcepts() + )) + .sum(), + // total size of entries + namespace.getStorage() + .getAllImports() + .map(Id::resolve) + .mapToLong( + Import::estimateMemoryConsumption) + .sum() + ) ); } @@ -141,4 +155,4 @@ public static class DatasetInfos { private long cBlocksSize; private long size; } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/GroupUIResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/GroupUIResource.java index 594fc7802b1..9969a53b881 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/GroupUIResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/GroupUIResource.java @@ -30,8 +30,10 @@ public class GroupUIResource { @GET public View getGroups() { - return new UIView<>("groups.html.ftl", uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), uiProcessor.getAdminProcessor() - .getAllGroups() + return new UIView<>( + "groups.html.ftl", + uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), + uiProcessor.getAdminProcessor().getAllGroups() ); } @@ -44,9 +46,10 @@ public View getGroups() { @Path("{" + GROUP_ID + "}") @GET public View getGroup(@PathParam(GROUP_ID) GroupId group) { - return new UIView<>("group.html.ftl", - uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), - uiProcessor.getGroupContent(group.resolve()) + return new UIView<>( + "group.html.ftl", + uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), + uiProcessor.getGroupContent(group.resolve()) ); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/IndexServiceUIResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/IndexServiceUIResource.java index 1243d405269..858368964ab 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/IndexServiceUIResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/IndexServiceUIResource.java @@ -2,7 +2,6 @@ import static com.bakdata.conquery.resources.ResourceConstants.INDEX_SERVICE_PATH_ELEMENT; -import java.util.Set; import jakarta.inject.Inject; import jakarta.ws.rs.GET; import jakarta.ws.rs.Path; @@ -10,6 +9,7 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.MediaType; +import java.util.Set; import com.bakdata.conquery.models.auth.web.csrf.CsrfTokenSetFilter; import com.bakdata.conquery.models.index.IndexKey; @@ -32,8 +32,13 @@ public class IndexServiceUIResource { @GET @Path(INDEX_SERVICE_PATH_ELEMENT) public View getIndexService() { - final IndexServiceUIContent content = new IndexServiceUIContent(uiProcessor.getIndexServiceStatistics(), uiProcessor.getLoadedIndexes()); - return new UIView<>("indexService.html.ftl", uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), content); + final IndexServiceUIContent content = new IndexServiceUIContent( + uiProcessor.getIndexServiceStatistics(), + uiProcessor.getLoadedIndexes()); + return new UIView<>( + "indexService.html.ftl", + uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), + content); } /** diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/RoleUIResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/RoleUIResource.java index 9532d49b5cf..4ad2641c86c 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/RoleUIResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/RoleUIResource.java @@ -30,8 +30,10 @@ public class RoleUIResource { @GET public View getRoles() { - return new UIView<>("roles.html.ftl", uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), uiProcessor.getAdminProcessor() - .getAllRoles() + return new UIView<>( + "roles.html.ftl", + uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), + uiProcessor.getAdminProcessor().getAllRoles() ); } @@ -45,6 +47,9 @@ public View getRoles() { @Path("{" + ROLE_ID + "}") @GET public View getRole(@PathParam(ROLE_ID) RoleId role) { - return new UIView<>("role.html.ftl", uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), uiProcessor.getRoleContent(role)); + return new UIView<>( + "role.html.ftl", + uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), + uiProcessor.getRoleContent(role)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/TablesUIResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/TablesUIResource.java index 51523bf5a86..13de3a8cbaa 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/TablesUIResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/TablesUIResource.java @@ -34,13 +34,19 @@ public TablesUIResource(UIProcessor uiProcessor) { @GET public View getTableView(@PathParam(TABLE) TableId tableId) { - return new UIView<>("table.html.ftl", uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), uiProcessor.getTableStatistics(tableId)); + return new UIView<>( + "table.html.ftl", + uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), + uiProcessor.getTableStatistics(tableId)); } @GET @Path("import/{" + IMPORT_ID + "}") public View getImportView(@PathParam(IMPORT_ID) ImportId imp) { - return new UIView<>("import.html.ftl", uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), uiProcessor.getImportStatistics(imp)); + return new UIView<>( + "import.html.ftl", + uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), + uiProcessor.getImportStatistics(imp)); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/UserUIResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/UserUIResource.java index 091310c9c64..5b1c070b415 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/UserUIResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/UserUIResource.java @@ -30,20 +30,25 @@ public class UserUIResource { @GET public View getUsers() { - return new UIView<>("users.html.ftl", uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), uiProcessor.getAdminProcessor() - .getAllUsers() + return new UIView<>( + "users.html.ftl", + uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), + uiProcessor.getAdminProcessor().getAllUsers() ); } /** * End point for retrieving information about a specific user. - * + * * @param user Unique id of the user. * @return A view holding the information about the user. */ @Path("{" + USER_ID + "}") @GET public View getUser(@PathParam(USER_ID) UserId user) { - return new UIView<>("user.html.ftl", uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), uiProcessor.getUserContent(user)); + return new UIView<>( + "user.html.ftl", + uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), + uiProcessor.getUserContent(user)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/ConnectorUIResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/ConnectorUIResource.java index f7ec5753011..6b0788827c6 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/ConnectorUIResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/ConnectorUIResource.java @@ -44,9 +44,9 @@ public class ConnectorUIResource { @GET public View getConnectorView() { return new UIView<>( - "connector.html.ftl", - uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), - connector.resolve() + "connector.html.ftl", + uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), + connector.resolve() ); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/FrontendAuthOverview.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/FrontendAuthOverview.java index 8fd78887c1b..3a48408c5d7 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/FrontendAuthOverview.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/FrontendAuthOverview.java @@ -22,7 +22,7 @@ public static class OverviewRow implements Comparable { private User user; private Collection groups; private Collection effectiveRoles; - + @Override public int compareTo(OverviewRow o) { return user.compareTo(o.getUser()); diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/FrontendGroupContent.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/FrontendGroupContent.java index c07d2f9b82c..ca69390110a 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/FrontendGroupContent.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/FrontendGroupContent.java @@ -20,4 +20,4 @@ public class FrontendGroupContent extends FrontendPermissionOwnerContent public final Collection roles; public final Collection availableRoles; -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/FrontendPermission.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/FrontendPermission.java index d0560587733..e7c50a24570 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/FrontendPermission.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/FrontendPermission.java @@ -27,11 +27,11 @@ public class FrontendPermission implements Comparable { public static FrontendPermission from(ConqueryPermission cPermission) { return new FrontendPermission( - cPermission.getDomains(), - cPermission.getAbilities(), - cPermission.getInstances(), - LocalDateTime.ofInstant(cPermission.getCreationTime(), TIMEZONE).format(DateTimeFormatter.ISO_DATE_TIME), - cPermission.toString() + cPermission.getDomains(), + cPermission.getAbilities(), + cPermission.getInstances(), + LocalDateTime.ofInstant(cPermission.getCreationTime(), TIMEZONE).format(DateTimeFormatter.ISO_DATE_TIME), + cPermission.toString() ); } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/FrontendRoleContent.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/FrontendRoleContent.java index 979928ce678..553ca282a37 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/FrontendRoleContent.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/FrontendRoleContent.java @@ -14,4 +14,4 @@ public class FrontendRoleContent extends FrontendPermissionOwnerContent { private List users; private List groups; -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/FrontendUserContent.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/FrontendUserContent.java index 739550e6f8f..81c7c2ac77d 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/FrontendUserContent.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/FrontendUserContent.java @@ -19,4 +19,4 @@ public class FrontendUserContent extends FrontendPermissionOwnerContent im public final Collection availableRoles; public final Collection groups; public final Collection availableGroups; -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/ImportStatistics.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/ImportStatistics.java index 8e9d7c8f6c4..60e4e52ec03 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/ImportStatistics.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/ImportStatistics.java @@ -1,15 +1,14 @@ package com.bakdata.conquery.resources.admin.ui.model; -import java.util.List; import com.bakdata.conquery.models.datasets.Import; -import com.bakdata.conquery.models.datasets.Table; import lombok.Getter; import lombok.RequiredArgsConstructor; -@Getter @RequiredArgsConstructor +@Getter +@RequiredArgsConstructor public class ImportStatistics { - + private final Import imp; private final long cBlocksMemoryBytes; } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/TableStatistics.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/TableStatistics.java index d3039adcebf..c4d63e605ac 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/TableStatistics.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/TableStatistics.java @@ -9,9 +9,10 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; -@Getter @RequiredArgsConstructor +@Getter +@RequiredArgsConstructor public class TableStatistics { - + private final Table table; private final long numberOfEntries; private final long size; diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/UIContext.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/UIContext.java index ef6fe0ccd77..329ebcc6422 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/UIContext.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/UIContext.java @@ -27,9 +27,12 @@ public class UIContext { public final String csrfToken; public Map getShardNodes() { - return shardNodeSupplier.get().stream().collect(Collectors.toMap( - ShardNodeInformation::getRemoteAddress, - Function.identity() - )); + return shardNodeSupplier.get() + .stream() + .collect( + Collectors.toMap( + ShardNodeInformation::getRemoteAddress, + Function.identity() + )); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/UIView.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/UIView.java index a4b360d0db1..8ccc22043d2 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/UIView.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/UIView.java @@ -7,14 +7,14 @@ @Getter public class UIView extends View { - + private final UIContext ctx; private final CONTENT_TYPE c; - + public UIView(String templateName, UIContext ctx) { this(templateName, ctx, null); } - + public UIView(String templateName, UIContext ctx, CONTENT_TYPE content) { super(resolve(templateName), StandardCharsets.UTF_8); this.c = content; @@ -22,6 +22,6 @@ public UIView(String templateName, UIContext ctx, CONTENT_TYPE content) { } private static String resolve(String templateName) { - return String.format("/com/bakdata/conquery/resources/admin/ui/%s", templateName); + return String.format("/com/bakdata/conquery/resources/admin/ui/%s", templateName); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptResource.java index e775aa6ebd1..ad7e0485d8f 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptResource.java @@ -2,8 +2,6 @@ import static com.bakdata.conquery.resources.ResourceConstants.CONCEPT; -import java.util.List; -import java.util.stream.Collectors; import jakarta.inject.Inject; import jakarta.servlet.http.HttpServletResponse; import jakarta.ws.rs.Consumes; @@ -16,6 +14,8 @@ import jakarta.ws.rs.core.EntityTag; import jakarta.ws.rs.core.HttpHeaders; import jakarta.ws.rs.core.Response; +import java.util.List; +import java.util.stream.Collectors; import com.bakdata.conquery.apiv1.frontend.FrontendList; import com.bakdata.conquery.io.jersey.ExtraMimeTypes; @@ -45,7 +45,8 @@ public Response getNode(@PathParam(CONCEPT) ConceptId concept) { // check if browser still has this version cached if (request.getHeaderString(HttpHeaders.IF_NONE_MATCH) != null && result.getCacheId() - .equals(EntityTag.valueOf(request.getHeaderString(HttpHeaders.IF_NONE_MATCH)))) { + .equals( + EntityTag.valueOf(request.getHeaderString(HttpHeaders.IF_NONE_MATCH)))) { return Response.status(HttpServletResponse.SC_NOT_MODIFIED).build(); } return Response.ok(result).tag(result.getCacheId()).build(); @@ -54,7 +55,9 @@ public Response getNode(@PathParam(CONCEPT) ConceptId concept) { @POST @Path("resolve") - public ConceptsProcessor.ResolvedConceptsResult resolve(@PathParam(CONCEPT) ConceptId conceptId, ConceptResource.ConceptCodeList conceptCodes) { + public ConceptsProcessor.ResolvedConceptsResult resolve( + @PathParam(CONCEPT) ConceptId conceptId, + ConceptResource.ConceptCodeList conceptCodes) { final Concept concept = conceptId.resolve(); subject.authorize(concept.getDataset(), Ability.READ); diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java index c4a90828b29..0c801397ba4 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.resources.api; +import jakarta.inject.Inject; +import jakarta.validation.Validator; import java.util.Collection; import java.util.Collections; import java.util.Comparator; @@ -11,8 +13,6 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; -import jakarta.inject.Inject; -import jakarta.validation.Validator; import com.bakdata.conquery.apiv1.IdLabel; import com.bakdata.conquery.apiv1.frontend.FrontendList; @@ -62,13 +62,17 @@ public class ConceptsProcessor { @Getter(lazy = true) private final FrontEndConceptBuilder frontEndConceptBuilder = new FrontEndConceptBuilder(getConfig()); - private final LoadingCache, FrontendList> nodeCache = - CacheBuilder.newBuilder().softValues().expireAfterWrite(10, TimeUnit.MINUTES).build(new CacheLoader<>() { - @Override - public FrontendList load(Concept concept) { - return getFrontEndConceptBuilder().createTreeMap(concept); - } - }); + private final LoadingCache, FrontendList> nodeCache = CacheBuilder.newBuilder() + .softValues() + .expireAfterWrite( + 10, + TimeUnit.MINUTES) + .build(new CacheLoader<>() { + @Override + public FrontendList load(Concept concept) { + return getFrontEndConceptBuilder().createTreeMap(concept); + } + }); public FrontendRoot getRoot(NamespaceStorage storage, Subject subject, boolean showHidden) { @@ -84,18 +88,19 @@ public FrontendRoot getRoot(NamespaceStorage storage, Subject subject, boolean s public FrontendList getNode(Concept concept) { try { return nodeCache.get(concept); - } - catch (ExecutionException e) { + } catch (ExecutionException e) { throw new RuntimeException("failed to create frontend node for " + concept, e); } } public Stream> getDatasets(Subject subject) { return namespaces.getAllDatasets() - .filter(d -> subject.isPermitted(d, Ability.READ)) - .map(DatasetId::resolve) - .sorted(Comparator.comparing(Dataset::getWeight).thenComparing(Dataset::getLabel)) - .map(d -> new IdLabel<>(d.getId(), d.getLabel())); + .filter(d -> subject.isPermitted(d, Ability.READ)) + .map( + DatasetId::resolve) + .sorted(Comparator.comparing(Dataset::getWeight).thenComparing(Dataset::getLabel)) + .map( + d -> new IdLabel<>(d.getId(), d.getLabel())); } public FrontendPreviewConfig getEntityPreviewFrontendConfig(DatasetId dataset) { @@ -107,18 +112,24 @@ public FrontendPreviewConfig getEntityPreviewFrontendConfig(DatasetId dataset) { ConnectorId searchConnector = previewConfig.resolveSearchConnector(); return new FrontendPreviewConfig( - previewConfig.getAllConnectors() - .stream() - .map(id -> new FrontendPreviewConfig.Labelled(id.toString(), id.resolve().getResolvedTable().getLabel())) - .collect(Collectors.toSet()), - - previewConfig.getDefaultConnectors() - .stream() - .map(id -> new FrontendPreviewConfig.Labelled(id.toString(), id.resolve().getResolvedTable().getLabel())) - .collect(Collectors.toSet()), - previewConfig.getSearchFilters(), - searchConnector.getConcept(), - searchConnector + previewConfig.getAllConnectors() + .stream() + .map( + id -> new FrontendPreviewConfig.Labelled( + id.toString(), + id.resolve().getResolvedTable().getLabel())) + .collect(Collectors.toSet()), + + previewConfig.getDefaultConnectors() + .stream() + .map( + id -> new FrontendPreviewConfig.Labelled( + id.toString(), + id.resolve().getResolvedTable().getLabel())) + .collect(Collectors.toSet()), + previewConfig.getSearchFilters(), + searchConnector.getConcept(), + searchConnector ); } @@ -132,14 +143,16 @@ public ResolvedFilterValues resolveFilterValues(FilterId filterId, List final ExactFilterValueResult exactResult = namespace.getFilterSearch().findExact(filter, searchTerms); - return new ResolvedFilterValues(new ResolvedFilterResult(filterId.getConnector(), filter.getId().toString(), exactResult.resolved), exactResult.unresolved); + return new ResolvedFilterValues( + new ResolvedFilterResult(filterId.getConnector(), filter.getId().toString(), exactResult.resolved), + exactResult.unresolved); } public AutoCompleteResult autocompleteTextFilter( - FilterId filterId, - String maybeText, - OptionalInt pageNumberOpt, - OptionalInt itemsPerPageOpt + FilterId filterId, + String maybeText, + OptionalInt pageNumberOpt, + OptionalInt itemsPerPageOpt ) { final int pageNumber = pageNumberOpt.orElse(0); final int itemsPerPage = itemsPerPageOpt.orElse(50); @@ -149,7 +162,12 @@ public AutoCompleteResult autocompleteTextFilter( final SelectFilter filter = (SelectFilter) filterId.resolve(); - log.trace("Searching for for `{}` in `{}`. (Page = {}, Items = {})", maybeText, filterId, pageNumber, itemsPerPage); + log.trace( + "Searching for for `{}` in `{}`. (Page = {}, Items = {})", + maybeText, + filterId, + pageNumber, + itemsPerPage); return namespaces.get(filter.getDataset()).getFilterSearch().query(filter, maybeText, itemsPerPage, pageNumber); } @@ -161,16 +179,16 @@ public ResolvedConceptsResult resolveConceptElements(TreeConcept concept, List child = concept.findMostSpecificChild(conceptCode, new CalculatedValue<>(Collections::emptyMap)); + final ConceptElement child = concept.findMostSpecificChild( + conceptCode, + new CalculatedValue<>(Collections::emptyMap)); if (child != null) { resolvedCodes.add(child.getId()); - } - else { + } else { unknownCodes.add(conceptCode); } - } - catch (ConceptConfigurationException e) { + } catch (ConceptConfigurationException e) { log.error("Error while trying to resolve `{}`", conceptCode, e); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java index c07e3fbc671..f0f1128fb0f 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java @@ -1,5 +1,9 @@ package com.bakdata.conquery.resources.api; +import jakarta.inject.Inject; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; import java.time.Year; import com.bakdata.conquery.apiv1.AdditionalMediaTypes; @@ -8,10 +12,6 @@ import com.bakdata.conquery.models.config.FrontendConfig; import com.bakdata.conquery.models.config.IdColumnConfig; import com.bakdata.conquery.util.VersionInfo; -import jakarta.inject.Inject; -import jakarta.ws.rs.GET; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.Produces; import lombok.RequiredArgsConstructor; @Path("config") @@ -29,12 +29,12 @@ public FrontendConfiguration getFrontendConfig() { final FrontendConfig frontendConfig = config.getFrontend(); return new FrontendConfiguration( - VersionInfo.INSTANCE.getVersions(), - frontendConfig.getCurrency(), - idColumns, - frontendConfig.getManualUrl(), - frontendConfig.getContactEmail(), - Year.now().minusYears(frontendConfig.getObservationPeriodYears()).atDay(1) + VersionInfo.INSTANCE.getVersions(), + frontendConfig.getCurrency(), + idColumns, + frontendConfig.getManualUrl(), + frontendConfig.getContactEmail(), + Year.now().minusYears(frontendConfig.getObservationPeriodYears()).atDay(1) ); } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/DatasetFormConfigResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/DatasetFormConfigResource.java index b44decfd348..9085314cbaf 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/DatasetFormConfigResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/DatasetFormConfigResource.java @@ -2,8 +2,6 @@ import static com.bakdata.conquery.resources.ResourceConstants.DATASET; -import java.util.Set; -import java.util.stream.Stream; import jakarta.inject.Inject; import jakarta.validation.Valid; import jakarta.ws.rs.Consumes; @@ -15,6 +13,8 @@ import jakarta.ws.rs.QueryParam; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.Response.Status; +import java.util.Set; +import java.util.stream.Stream; import com.bakdata.conquery.apiv1.forms.FormConfigAPI; import com.bakdata.conquery.io.jersey.ExtraMimeTypes; @@ -44,11 +44,16 @@ public class DatasetFormConfigResource extends HAuthorized { public Response postConfig(@Auth Subject subject, @Valid FormConfigAPI config) { subject.authorize(dataset, Ability.READ); - return Response.ok(new PostResponse(processor.addConfig(subject, dataset, config).getId())).status(Status.CREATED).build(); + return Response.ok(new PostResponse(processor.addConfig(subject, dataset, config).getId())) + .status( + Status.CREATED) + .build(); } @GET - public Stream getConfigByUserAndType(@Auth Subject subject, @QueryParam("formType") Set formType) { + public Stream getConfigByUserAndType( + @Auth Subject subject, + @QueryParam("formType") Set formType) { subject.authorize(dataset, Ability.READ); return processor.getConfigsByFormType(subject, dataset, formType); diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/DatasetFormResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/DatasetFormResource.java index 8f9dbf0609a..30ff6ff4e5b 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/DatasetFormResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/DatasetFormResource.java @@ -2,6 +2,12 @@ import static com.bakdata.conquery.resources.ResourceConstants.DATASET; +import jakarta.inject.Inject; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; import java.util.Collection; import com.bakdata.conquery.apiv1.AdditionalMediaTypes; @@ -9,12 +15,6 @@ import com.bakdata.conquery.models.forms.frontendconfiguration.FormProcessor; import com.fasterxml.jackson.databind.JsonNode; import io.dropwizard.auth.Auth; -import jakarta.inject.Inject; -import jakarta.ws.rs.Consumes; -import jakarta.ws.rs.GET; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.Produces; -import jakarta.ws.rs.core.MediaType; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/DatasetQueryResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/DatasetQueryResource.java index 7b7845600d1..cb229d585da 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/DatasetQueryResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/DatasetQueryResource.java @@ -2,10 +2,6 @@ import static com.bakdata.conquery.resources.ResourceConstants.DATASET; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Stream; import jakarta.inject.Inject; import jakarta.servlet.http.HttpServletRequest; import jakarta.validation.Valid; @@ -21,6 +17,10 @@ import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.UriBuilder; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; import com.bakdata.conquery.apiv1.AdditionalMediaTypes; import com.bakdata.conquery.apiv1.QueryProcessor; @@ -56,18 +56,30 @@ public class DatasetQueryResource { @POST @Path("/entity") - public FullExecutionStatus getEntityData(@Auth Subject subject, @Valid EntityPreviewRequest query, @Context HttpServletRequest request) { + public FullExecutionStatus getEntityData( + @Auth Subject subject, + @Valid EntityPreviewRequest query, + @Context HttpServletRequest request) { subject.authorize(dataset, Ability.READ); subject.authorize(dataset, Ability.PRESERVE_ID); final UriBuilder uriBuilder = RequestAwareUriBuilder.fromRequest(request); - return processor.getSingleEntityExport(subject, uriBuilder, query.getIdKind(), query.getEntityId(), query.getSources(), dataset, query.getTime()); + return processor.getSingleEntityExport( + subject, + uriBuilder, + query.getIdKind(), + query.getEntityId(), + query.getSources(), + dataset, + query.getTime()); } @POST @Path("/resolve-entities") - public Stream> resolveEntities(@Auth Subject subject, @Valid @NotEmpty List> container) { + public Stream> resolveEntities( + @Auth Subject subject, + @Valid @NotEmpty List> container) { subject.authorize(dataset, Ability.READ); subject.authorize(dataset, Ability.PRESERVE_ID); @@ -75,7 +87,6 @@ public Stream> resolveEntities(@Auth Subject subject, @Valid } - @POST @Path("/upload") public ExternalUploadResult upload(@Auth Subject subject, @Valid ExternalUpload upload) { @@ -87,7 +98,9 @@ public ExternalUploadResult upload(@Auth Subject subject, @Valid ExternalUpload @GET - public List getAllQueries(@Auth Subject subject, @QueryParam("all-providers") Optional allProviders) { + public List getAllQueries( + @Auth Subject subject, + @QueryParam("all-providers") Optional allProviders) { subject.authorize(dataset, Ability.READ); @@ -95,19 +108,22 @@ public List getAllQueries(@Auth Subject subject, @Que } @POST - public Response postQuery(@Auth Subject subject, @QueryParam("all-providers") Optional allProviders, @NotNull @Valid QueryDescription query) { + public Response postQuery( + @Auth Subject subject, + @QueryParam("all-providers") Optional allProviders, + @NotNull @Valid QueryDescription query) { subject.authorize(dataset, Ability.READ); final ManagedExecution execution = processor.postQuery(dataset, query, subject, false); - return Response.ok(processor.getQueryFullStatus(execution.getId(), - subject, - RequestAwareUriBuilder.fromRequest(servletRequest), - allProviders.orElse(false), - false - )) - .status(Response.Status.CREATED) - .build(); + return Response.ok( + processor.getQueryFullStatus( + execution.getId(), + subject, + RequestAwareUriBuilder.fromRequest(servletRequest), + allProviders.orElse(false), + false + )).status(Response.Status.CREATED).build(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/DatasetsResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/DatasetsResource.java index 4de6959aea7..2f7d8123c55 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/DatasetsResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/DatasetsResource.java @@ -1,11 +1,11 @@ package com.bakdata.conquery.resources.api; -import java.util.stream.Stream; import jakarta.inject.Inject; import jakarta.ws.rs.Consumes; import jakarta.ws.rs.GET; import jakarta.ws.rs.Path; import jakarta.ws.rs.Produces; +import java.util.stream.Stream; import com.bakdata.conquery.apiv1.IdLabel; import com.bakdata.conquery.io.jersey.ExtraMimeTypes; diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/EntityPreviewRequest.java b/backend/src/main/java/com/bakdata/conquery/resources/api/EntityPreviewRequest.java index 895e5a220c2..819e96eec36 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/EntityPreviewRequest.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/EntityPreviewRequest.java @@ -1,8 +1,8 @@ package com.bakdata.conquery.resources.api; +import jakarta.validation.constraints.NotEmpty; import java.time.LocalDate; import java.util.List; -import jakarta.validation.constraints.NotEmpty; import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.identifiable.ids.specific.ConnectorId; diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/FilterResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/FilterResource.java index 2985fe0ed5d..d1208baf252 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/FilterResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/FilterResource.java @@ -2,9 +2,6 @@ import static com.bakdata.conquery.resources.ResourceConstants.FILTER; -import java.util.List; -import java.util.Optional; -import java.util.OptionalInt; import jakarta.inject.Inject; import jakarta.validation.Valid; import jakarta.validation.constraints.Max; @@ -17,6 +14,9 @@ import jakarta.ws.rs.Produces; import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.Response.Status; +import java.util.List; +import java.util.Optional; +import java.util.OptionalInt; import com.bakdata.conquery.io.jersey.ExtraMimeTypes; import com.bakdata.conquery.models.auth.permissions.Ability; @@ -57,19 +57,25 @@ public ConceptsProcessor.ResolvedFilterValues resolveFilterValues(FilterValues f //TODO migrate from filter to searchable @POST @Path("autocomplete") - public ConceptsProcessor.AutoCompleteResult autocompleteTextFilter(@Valid FilterResource.AutocompleteRequest request) { + public ConceptsProcessor.AutoCompleteResult autocompleteTextFilter( + @Valid FilterResource.AutocompleteRequest request) { subject.isPermitted(filter.getDataset(), Ability.READ); subject.isPermitted(filter.getConnector().getConcept(), Ability.READ); if (!(filter.resolve() instanceof SelectFilter)) { - throw new WebApplicationException(filter + " is not a SELECT filter, but " + filter.getClass().getSimpleName() + ".", Status.BAD_REQUEST); + throw new WebApplicationException( + filter + " is not a SELECT filter, but " + filter.getClass().getSimpleName() + ".", + Status.BAD_REQUEST); } try { - return processor.autocompleteTextFilter(filter, request.text().orElse(null), request.page(), request.pageSize()); - } - catch (IllegalArgumentException e) { + return processor.autocompleteTextFilter( + filter, + request.text().orElse(null), + request.page(), + request.pageSize()); + } catch (IllegalArgumentException e) { throw new BadRequestException(e); } } @@ -78,9 +84,9 @@ public record FilterValues(List values) { } public record AutocompleteRequest( - @NonNull Optional<@Size(max = MAX_AUTOCOMPLETE_TEXT_LENGTH) String> text, - @NonNull OptionalInt page, - @NonNull @Max(MAX_AUTOCOMPLETE_PAGE_SIZE) OptionalInt pageSize + @NonNull Optional<@Size(max = MAX_AUTOCOMPLETE_TEXT_LENGTH) String> text, + @NonNull OptionalInt page, + @NonNull @Max(MAX_AUTOCOMPLETE_PAGE_SIZE) OptionalInt pageSize ) { } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/FormConfigResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/FormConfigResource.java index f34dd45ce37..b6016bfd36f 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/FormConfigResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/FormConfigResource.java @@ -33,17 +33,20 @@ public class FormConfigResource { public FormConfigFullRepresentation getConfig(@Auth Subject subject, @PathParam(FORM_CONFIG) FormConfigId form) { return processor.getConfig(subject, form); } - + @PATCH @Path("{" + FORM_CONFIG + "}") - public FormConfigFullRepresentation patchConfig(@Auth Subject subject, @PathParam(FORM_CONFIG) FormConfigId form, FormConfigPatch patch) { + public FormConfigFullRepresentation patchConfig( + @Auth Subject subject, + @PathParam(FORM_CONFIG) FormConfigId form, + FormConfigPatch patch) { return processor.patchConfig(subject, form, patch); } - + @DELETE @Path("{" + FORM_CONFIG + "}") public void deleteConfig(@Auth Subject subject, @PathParam(FORM_CONFIG) FormConfigId form) { processor.deleteConfig(subject, form); } - + } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java index d912541678b..63db4a1bacd 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java @@ -44,14 +44,19 @@ public class QueryResource { @GET @Path("{" + QUERY + "}") public FullExecutionStatus getStatus( - @Auth Subject subject, - @PathParam(QUERY) ManagedExecutionId queryId, - @QueryParam("all-providers") @DefaultValue("false") boolean allProviders) { + @Auth Subject subject, + @PathParam(QUERY) ManagedExecutionId queryId, + @QueryParam("all-providers") @DefaultValue("false") boolean allProviders) { subject.authorize(queryId.getDataset(), Ability.READ); subject.authorize(queryId, Ability.READ); - return processor.getQueryFullStatus(queryId, subject, RequestAwareUriBuilder.fromRequest(servletRequest), allProviders, true); + return processor.getQueryFullStatus( + queryId, + subject, + RequestAwareUriBuilder.fromRequest(servletRequest), + allProviders, + true); } @GET @@ -66,16 +71,21 @@ public ResultStatistics getDescription(@Auth Subject subject, @PathParam(QUERY) @PATCH @Path("{" + QUERY + "}") public FullExecutionStatus patchQuery( - @Auth Subject subject, - @PathParam(QUERY) ManagedExecutionId query, - @QueryParam("all-providers") @DefaultValue("false") boolean allProviders, - MetaDataPatch patch) { + @Auth Subject subject, + @PathParam(QUERY) ManagedExecutionId query, + @QueryParam("all-providers") @DefaultValue("false") boolean allProviders, + MetaDataPatch patch) { subject.authorize(query.getDataset(), Ability.READ); subject.authorize(query, Ability.READ); processor.patchQuery(subject, query, patch); - return processor.getQueryFullStatus(query, subject, RequestAwareUriBuilder.fromRequest(servletRequest), allProviders, false); + return processor.getQueryFullStatus( + query, + subject, + RequestAwareUriBuilder.fromRequest(servletRequest), + allProviders, + false); } @DELETE @@ -90,14 +100,19 @@ public void deleteQuery(@Auth Subject subject, @PathParam(QUERY) ManagedExecutio @POST @Path("{" + QUERY + "}/reexecute") public FullExecutionStatus reexecute( - @Auth Subject subject, - @PathParam(QUERY) ManagedExecutionId query, - @QueryParam("all-providers") @DefaultValue("false") boolean allProviders) { + @Auth Subject subject, + @PathParam(QUERY) ManagedExecutionId query, + @QueryParam("all-providers") @DefaultValue("false") boolean allProviders) { subject.authorize(query.getDataset(), Ability.READ); subject.authorize(query, Ability.READ); processor.reexecute(subject, query); - return processor.getQueryFullStatus(query, subject, RequestAwareUriBuilder.fromRequest(servletRequest), allProviders, false); + return processor.getQueryFullStatus( + query, + subject, + RequestAwareUriBuilder.fromRequest(servletRequest), + allProviders, + false); } @POST diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowResource.java index afe154440db..e4bb9ec3a05 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowResource.java @@ -3,10 +3,6 @@ import static com.bakdata.conquery.io.result.ResultUtil.checkSingleTableResult; import static com.bakdata.conquery.resources.ResourceConstants.*; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.Optional; -import java.util.OptionalLong; import jakarta.inject.Inject; import jakarta.ws.rs.DefaultValue; import jakarta.ws.rs.GET; @@ -18,6 +14,10 @@ import jakarta.ws.rs.core.HttpHeaders; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.UriBuilder; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Optional; +import java.util.OptionalLong; import com.bakdata.conquery.apiv1.AdditionalMediaTypes; import com.bakdata.conquery.io.result.arrow.ResultArrowProcessor; @@ -41,51 +41,71 @@ public class ResultArrowResource { @Path("{" + QUERY + "}." + FILE_EXTENTION_ARROW_FILE) @Produces(AdditionalMediaTypes.ARROW_FILE) public Response getFile( - @Auth Subject subject, - @PathParam(QUERY) ManagedExecutionId query, - @HeaderParam(HttpHeaders.USER_AGENT) String userAgent, - @QueryParam("pretty") @DefaultValue("false") boolean pretty, - @QueryParam("limit") OptionalLong limit - ) { + @Auth Subject subject, + @PathParam(QUERY) ManagedExecutionId query, + @HeaderParam(HttpHeaders.USER_AGENT) String userAgent, + @QueryParam("pretty") @DefaultValue("false") boolean pretty, + @QueryParam("limit") OptionalLong limit + ) { checkSingleTableResult(query.resolve()); - log.info("Result for {} download on dataset {} by subject {} ({}).", query, query.getDataset(), subject.getId(), subject.getName()); + log.info( + "Result for {} download on dataset {} by subject {} ({}).", + query, + query.getDataset(), + subject.getId(), + subject.getName()); return processor.createResultFile(subject, query, pretty, limit); } - public static URL getFileDownloadURL(UriBuilder uriBuilder, E exec) throws MalformedURLException { - return uriBuilder - .path(ResultArrowResource.class) - .resolveTemplate(ResourceConstants.DATASET, exec.getDataset().getName()) - .path(ResultArrowResource.class, "getFile") - .resolveTemplate(ResourceConstants.QUERY, exec.getId().toString()) - .build() - .toURL(); + public static URL getFileDownloadURL( + UriBuilder uriBuilder, + E exec) throws MalformedURLException { + return uriBuilder.path(ResultArrowResource.class) + .resolveTemplate( + ResourceConstants.DATASET, + exec.getDataset().getName()) + .path(ResultArrowResource.class, "getFile") + .resolveTemplate( + ResourceConstants.QUERY, + exec.getId().toString()) + .build() + .toURL(); } - public static URL getStreamDownloadURL(UriBuilder uriBuilder, E exec) throws MalformedURLException { - return uriBuilder - .path(ResultArrowResource.class) - .resolveTemplate(ResourceConstants.DATASET, exec.getDataset().getName()) - .path(ResultArrowResource.class, "getStream") - .resolveTemplate(ResourceConstants.QUERY, exec.getId().toString()) - .build() - .toURL(); + public static URL getStreamDownloadURL( + UriBuilder uriBuilder, + E exec) throws MalformedURLException { + return uriBuilder.path(ResultArrowResource.class) + .resolveTemplate( + ResourceConstants.DATASET, + exec.getDataset().getName()) + .path(ResultArrowResource.class, "getStream") + .resolveTemplate( + ResourceConstants.QUERY, + exec.getId().toString()) + .build() + .toURL(); } @GET @Path("{" + QUERY + "}." + FILE_EXTENTION_ARROW_STREAM) @Produces(AdditionalMediaTypes.ARROW_STREAM) public Response getStream( - @Auth Subject subject, - @PathParam(QUERY) ManagedExecutionId execution, - @HeaderParam(HttpHeaders.USER_AGENT) String userAgent, - @QueryParam("pretty") Optional pretty, - @QueryParam("limit") OptionalLong limit + @Auth Subject subject, + @PathParam(QUERY) ManagedExecutionId execution, + @HeaderParam(HttpHeaders.USER_AGENT) String userAgent, + @QueryParam("pretty") Optional pretty, + @QueryParam("limit") OptionalLong limit ) { checkSingleTableResult(execution.resolve()); - log.info("Result for {} download on dataset {} by subject {} ({}).", execution, execution.getDataset(), subject.getId(), subject.getName()); + log.info( + "Result for {} download on dataset {} by subject {} ({}).", + execution, + execution.getDataset(), + subject.getId(), + subject.getName()); return processor.createResultStream(subject, execution, pretty.orElse(false), limit); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultCsvResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultCsvResource.java index 1759e9afa7c..5f74da2bcac 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultCsvResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultCsvResource.java @@ -4,9 +4,6 @@ import static com.bakdata.conquery.io.result.ResultUtil.determineCharset; import static com.bakdata.conquery.resources.ResourceConstants.QUERY; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.OptionalLong; import jakarta.inject.Inject; import jakarta.ws.rs.DefaultValue; import jakarta.ws.rs.GET; @@ -18,6 +15,9 @@ import jakarta.ws.rs.core.HttpHeaders; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.UriBuilder; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.OptionalLong; import com.bakdata.conquery.apiv1.AdditionalMediaTypes; import com.bakdata.conquery.io.result.csv.ResultCsvProcessor; @@ -39,30 +39,44 @@ public class ResultCsvResource { private final ResultCsvProcessor processor; - public static URL getDownloadURL(UriBuilder uriBuilder, E exec) throws MalformedURLException { + public static URL getDownloadURL( + UriBuilder uriBuilder, + E exec) throws MalformedURLException { return uriBuilder.path(ResultCsvResource.class) - .path(ResultCsvResource.class, GET_RESULT_PATH_METHOD) - .resolveTemplate(ResourceConstants.QUERY, exec.getId().toString()) - .build() - .toURL(); + .path( + ResultCsvResource.class, + GET_RESULT_PATH_METHOD) + .resolveTemplate(ResourceConstants.QUERY, exec.getId().toString()) + .build() + .toURL(); } @GET @Path("{" + QUERY + "}.csv") @Produces(AdditionalMediaTypes.CSV) public Response getAsCsv( - @Auth Subject subject, - @PathParam(QUERY) ManagedExecutionId executionId, - @HeaderParam(HttpHeaders.USER_AGENT) String userAgent, - @QueryParam("charset") String queryCharset, - @QueryParam("pretty") @DefaultValue("true") boolean pretty, - @QueryParam("limit") OptionalLong limit + @Auth Subject subject, + @PathParam(QUERY) ManagedExecutionId executionId, + @HeaderParam(HttpHeaders.USER_AGENT) String userAgent, + @QueryParam("charset") String queryCharset, + @QueryParam("pretty") @DefaultValue("true") boolean pretty, + @QueryParam("limit") OptionalLong limit ) { ManagedExecution execution = executionId.resolve(); checkSingleTableResult(execution); - log.info("Result for {} download on dataset {} by subject {} ({}).", executionId, execution.getDataset(), subject.getId(), subject.getName()); + log.info( + "Result for {} download on dataset {} by subject {} ({}).", + executionId, + execution.getDataset(), + subject.getId(), + subject.getName()); - return processor.createResult((ManagedExecution & SingleTableResult) execution, subject, pretty, determineCharset(userAgent, queryCharset), limit); + return processor.createResult( + (ManagedExecution & SingleTableResult) execution, + subject, + pretty, + determineCharset(userAgent, queryCharset), + limit); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExcelResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExcelResource.java index ca979343cdd..24970749afa 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExcelResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExcelResource.java @@ -3,9 +3,6 @@ import static com.bakdata.conquery.io.result.ResultUtil.checkSingleTableResult; import static com.bakdata.conquery.resources.ResourceConstants.QUERY; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.OptionalLong; import jakarta.inject.Inject; import jakarta.ws.rs.DefaultValue; import jakarta.ws.rs.GET; @@ -17,6 +14,9 @@ import jakarta.ws.rs.core.HttpHeaders; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.UriBuilder; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.OptionalLong; import com.bakdata.conquery.apiv1.AdditionalMediaTypes; import com.bakdata.conquery.io.result.excel.ResultExcelProcessor; @@ -42,23 +42,33 @@ public class ResultExcelResource { @Path("{" + QUERY + "}.xlsx") @Produces(AdditionalMediaTypes.EXCEL) public Response get( - @Auth Subject subject, - @PathParam(QUERY) ManagedExecutionId execution, - @HeaderParam(HttpHeaders.USER_AGENT) String userAgent, - @QueryParam("pretty") @DefaultValue("true") boolean pretty, - @QueryParam("limit") OptionalLong limit) { + @Auth Subject subject, + @PathParam(QUERY) ManagedExecutionId execution, + @HeaderParam(HttpHeaders.USER_AGENT) String userAgent, + @QueryParam("pretty") @DefaultValue("true") boolean pretty, + @QueryParam("limit") OptionalLong limit) { checkSingleTableResult(execution.resolve()); - log.info("Result for {} download on dataset {} by subject {} ({}).", execution, execution.getDataset(), subject.getId(), subject.getName()); + log.info( + "Result for {} download on dataset {} by subject {} ({}).", + execution, + execution.getDataset(), + subject.getId(), + subject.getName()); return processor.createResult(subject, execution, pretty, limit); } - public static URL getDownloadURL(UriBuilder uriBuilder, E exec) throws MalformedURLException { - return uriBuilder - .path(ResultExcelResource.class) - .resolveTemplate(ResourceConstants.DATASET, exec.getDataset().getName()) - .path(ResultExcelResource.class, GET_RESULT_PATH_METHOD) - .resolveTemplate(ResourceConstants.QUERY, exec.getId().toString()) - .build() - .toURL(); + public static URL getDownloadURL( + UriBuilder uriBuilder, + E exec) throws MalformedURLException { + return uriBuilder.path(ResultExcelResource.class) + .resolveTemplate( + ResourceConstants.DATASET, + exec.getDataset().getName()) + .path(ResultExcelResource.class, GET_RESULT_PATH_METHOD) + .resolveTemplate( + ResourceConstants.QUERY, + exec.getId().toString()) + .build() + .toURL(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExternalResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExternalResource.java index 8385f7e788d..17ff949a3b0 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExternalResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExternalResource.java @@ -4,8 +4,6 @@ import static com.bakdata.conquery.resources.ResourceConstants.FILENAME; import static com.bakdata.conquery.resources.ResourceConstants.QUERY; -import java.net.URI; -import java.net.URISyntaxException; import jakarta.inject.Inject; import jakarta.ws.rs.GET; import jakarta.ws.rs.HeaderParam; @@ -14,6 +12,8 @@ import jakarta.ws.rs.QueryParam; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.UriBuilder; +import java.net.URI; +import java.net.URISyntaxException; import com.bakdata.conquery.io.result.external.ExternalResultProcessor; import com.bakdata.conquery.models.auth.entities.Subject; @@ -32,14 +32,19 @@ public class ResultExternalResource { private ExternalResultProcessor processor; - public static URI getDownloadURL(UriBuilder uriBuilder, ExternalExecution exec, String filename) - throws URISyntaxException { - return uriBuilder - .path(ResultExternalResource.class) - .path(ResultExternalResource.class, DOWNLOAD_PATH_METHOD) - .resolveTemplate(ResourceConstants.QUERY, exec.getId().toString()) - .resolveTemplate(FILENAME, filename) - .build(); + public static URI getDownloadURL( + UriBuilder uriBuilder, + ExternalExecution exec, + String filename) throws URISyntaxException { + return uriBuilder.path(ResultExternalResource.class) + .path( + ResultExternalResource.class, + DOWNLOAD_PATH_METHOD) + .resolveTemplate(ResourceConstants.QUERY, exec.getId().toString()) + .resolveTemplate( + FILENAME, + filename) + .build(); } @@ -56,13 +61,18 @@ public static URI getDownloadURL(UriBuilder uriBuilder, ExternalExecution exec, @GET @Path("{" + QUERY + "}/{" + FILENAME + "}") public Response download( - @Auth Subject subject, - @PathParam(QUERY) ManagedExecutionId execution, - @PathParam(FILENAME) String fileName, - @HeaderParam("user-agent") String userAgent, - @QueryParam("charset") String queryCharset + @Auth Subject subject, + @PathParam(QUERY) ManagedExecutionId execution, + @PathParam(FILENAME) String fileName, + @HeaderParam("user-agent") String userAgent, + @QueryParam("charset") String queryCharset ) { - log.info("Result download for {} on dataset {} by user {} ({}).", execution, execution.getDataset(), subject.getId(), subject.getName()); + log.info( + "Result download for {} on dataset {} by user {} ({}).", + execution, + execution.getDataset(), + subject.getId(), + subject.getName()); return processor.getResult(subject, execution, fileName); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultJsonDescriptionResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultJsonDescriptionResource.java index 815a3c54301..3cba698b29f 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultJsonDescriptionResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultJsonDescriptionResource.java @@ -2,8 +2,6 @@ import static com.bakdata.conquery.resources.ResourceConstants.QUERY; -import java.net.MalformedURLException; -import java.net.URL; import jakarta.inject.Inject; import jakarta.ws.rs.GET; import jakarta.ws.rs.Path; @@ -12,6 +10,8 @@ import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.UriBuilder; +import java.net.MalformedURLException; +import java.net.URL; import com.bakdata.conquery.io.result.json.ResultJsonDescriptionProcessor; import com.bakdata.conquery.models.auth.entities.Subject; @@ -33,7 +33,12 @@ public class ResultJsonDescriptionResource { @Produces(MediaType.APPLICATION_JSON) public Response getAsJson(@Auth Subject subject, @PathParam(QUERY) ManagedExecutionId execution) { - log.debug("Result for {} download on dataset {} by subject {} ({}).", execution, execution.getDataset(), subject.getId(), subject.getName()); + log.debug( + "Result for {} download on dataset {} by subject {} ({}).", + execution, + execution.getDataset(), + subject.getId(), + subject.getName()); return processor.createResult(subject, execution); } @@ -41,9 +46,11 @@ public Response getAsJson(@Auth Subject subject, @PathParam(QUERY) ManagedExecut public static URL getDownloadURL(UriBuilder uriBuilder, ManagedExecutionId execId) throws MalformedURLException { return uriBuilder.path(ResultJsonDescriptionResource.class) - .path(ResultJsonDescriptionResource.class, "getAsJson") - .resolveTemplate(ResourceConstants.QUERY, execId.toString()) - .build() - .toURL(); + .path( + ResultJsonDescriptionResource.class, + "getAsJson") + .resolveTemplate(ResourceConstants.QUERY, execId.toString()) + .build() + .toURL(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultParquetResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultParquetResource.java index ff7a0e8e299..32e9b058cc3 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultParquetResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultParquetResource.java @@ -4,9 +4,6 @@ import static com.bakdata.conquery.resources.ResourceConstants.FILE_EXTENTION_PARQUET; import static com.bakdata.conquery.resources.ResourceConstants.QUERY; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.OptionalLong; import jakarta.inject.Inject; import jakarta.ws.rs.DefaultValue; import jakarta.ws.rs.GET; @@ -18,6 +15,9 @@ import jakarta.ws.rs.core.HttpHeaders; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.UriBuilder; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.OptionalLong; import com.bakdata.conquery.io.result.parquet.ResultParquetProcessor; import com.bakdata.conquery.models.auth.entities.Subject; @@ -38,28 +38,38 @@ public class ResultParquetResource { private final ResultParquetProcessor processor; - public static URL getDownloadURL(UriBuilder uriBuilder, E exec) throws MalformedURLException { - return uriBuilder - .path(ResultParquetResource.class) - .resolveTemplate(ResourceConstants.DATASET, exec.getDataset().getName()) - .path(ResultParquetResource.class, "getFile") - .resolveTemplate(ResourceConstants.QUERY, exec.getId().toString()) - .build() - .toURL(); + public static URL getDownloadURL( + UriBuilder uriBuilder, + E exec) throws MalformedURLException { + return uriBuilder.path(ResultParquetResource.class) + .resolveTemplate( + ResourceConstants.DATASET, + exec.getDataset().getName()) + .path(ResultParquetResource.class, "getFile") + .resolveTemplate( + ResourceConstants.QUERY, + exec.getId().toString()) + .build() + .toURL(); } @GET @Path("{" + QUERY + "}." + FILE_EXTENTION_PARQUET) @Produces(PARQUET_MEDIA_TYPE_STRING) public Response getFile( - @Auth Subject subject, - @PathParam(QUERY) ManagedExecutionId execution, - @HeaderParam(HttpHeaders.USER_AGENT) String userAgent, - @QueryParam("pretty") @DefaultValue("false") boolean pretty, - @QueryParam("limit") OptionalLong limit) { + @Auth Subject subject, + @PathParam(QUERY) ManagedExecutionId execution, + @HeaderParam(HttpHeaders.USER_AGENT) String userAgent, + @QueryParam("pretty") @DefaultValue("false") boolean pretty, + @QueryParam("limit") OptionalLong limit) { checkSingleTableResult(execution.resolve()); - log.info("Result for {} download on dataset {} by subject {} ({}).", execution, execution.getDataset(), subject.getId(), subject.getName()); + log.info( + "Result for {} download on dataset {} by subject {} ({}).", + execution, + execution.getDataset(), + subject.getId(), + subject.getName()); return processor.createResultFile(subject, execution, pretty, limit); } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/hierarchies/HAuthorized.java b/backend/src/main/java/com/bakdata/conquery/resources/hierarchies/HAuthorized.java index 8f61179e651..ddda067fbe7 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/hierarchies/HAuthorized.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/hierarchies/HAuthorized.java @@ -1,11 +1,10 @@ package com.bakdata.conquery.resources.hierarchies; -import java.security.Principal; - import jakarta.annotation.PostConstruct; import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.Response.Status; +import java.security.Principal; import com.bakdata.conquery.models.auth.entities.Subject; import lombok.Getter; diff --git a/backend/src/main/java/com/bakdata/conquery/resources/hierarchies/HierarchyHelper.java b/backend/src/main/java/com/bakdata/conquery/resources/hierarchies/HierarchyHelper.java index 532de433da3..e74a3cca893 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/hierarchies/HierarchyHelper.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/hierarchies/HierarchyHelper.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.resources.hierarchies; -import java.lang.reflect.Method; - import jakarta.ws.rs.DELETE; import jakarta.ws.rs.GET; import jakarta.ws.rs.HEAD; @@ -10,6 +8,7 @@ import jakarta.ws.rs.PUT; import jakarta.ws.rs.Path; import jakarta.ws.rs.core.UriBuilder; +import java.lang.reflect.Method; import lombok.experimental.UtilityClass; @@ -51,7 +50,7 @@ public static UriBuilder hierarchicalPath(UriBuilder uri, Class clazz, String throw new IllegalArgumentException( String.format("Method %s not found or is not annotated as HttpMethod in class %s", methodName, clazz)); } - + boolean foundPath = false; if (clazz.isAnnotationPresent(Path.class)) { @@ -63,20 +62,19 @@ public static UriBuilder hierarchicalPath(UriBuilder uri, Class clazz, String uri.path(clazz, methodName); foundPath = true; } - + if (!foundPath) { - throw new IllegalArgumentException("The javax.ws.rs.Path annotation was present neither on the class '"+ clazz.getName() + "' nor on the method " + methodName + "'."); + throw new IllegalArgumentException( + "The javax.ws.rs.Path annotation was present neither on the class '" + clazz + .getName() + "' nor on the method " + methodName + "'."); } - + return uri; } private static boolean isEndpoint(Method method) { - return method.isAnnotationPresent(GET.class) - || method.isAnnotationPresent(DELETE.class) - || method.isAnnotationPresent(HEAD.class) - || method.isAnnotationPresent(OPTIONS.class) - || method.isAnnotationPresent(POST.class) - || method.isAnnotationPresent(PUT.class); + return method.isAnnotationPresent(GET.class) || method.isAnnotationPresent( + DELETE.class) || method.isAnnotationPresent(HEAD.class) || method.isAnnotationPresent( + OPTIONS.class) || method.isAnnotationPresent(POST.class) || method.isAnnotationPresent(PUT.class); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/unprotected/AuthServlet.java b/backend/src/main/java/com/bakdata/conquery/resources/unprotected/AuthServlet.java index e82d3807f1b..a48f7c0681b 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/unprotected/AuthServlet.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/unprotected/AuthServlet.java @@ -31,10 +31,14 @@ public class AuthServlet { /** * Prepares the general configuration with resources and settings that are valid * for both, api and admin, endpoints. - * + * * @return */ - public static DropwizardResourceConfig generalSetup(MetricRegistry metrics, ConqueryConfig config, ServletEnvironment servletEnvironment, ObjectMapper objectMapper) { + public static DropwizardResourceConfig generalSetup( + MetricRegistry metrics, + ConqueryConfig config, + ServletEnvironment servletEnvironment, + ObjectMapper objectMapper) { DropwizardResourceConfig jerseyConfig = new DropwizardResourceConfig(metrics); jerseyConfig.setUrlPattern("/auth"); @@ -46,7 +50,7 @@ public static DropwizardResourceConfig generalSetup(MetricRegistry metrics, Conq jerseyConfig.register(CORSPreflightRequestFilter.class); jerseyConfig.register(CORSResponseFilter.class); - + jerseyConfig.register(new JacksonMessageBodyProvider(objectMapper)); // freemarker support jerseyConfig.register(new ViewMessageBodyWriter(metrics, Collections.singleton(Freemarker.HTML_RENDERER))); diff --git a/backend/src/main/java/com/bakdata/conquery/resources/unprotected/LoginResource.java b/backend/src/main/java/com/bakdata/conquery/resources/unprotected/LoginResource.java index 945c4b70b9f..df736a6d10e 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/unprotected/LoginResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/unprotected/LoginResource.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.resources.unprotected; -import java.net.URI; import jakarta.ws.rs.BadRequestException; import jakarta.ws.rs.GET; import jakarta.ws.rs.Path; @@ -9,6 +8,7 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.MediaType; +import java.net.URI; import com.bakdata.conquery.models.auth.web.RedirectingAuthFilter; import com.bakdata.conquery.resources.admin.ui.model.UIView; @@ -20,13 +20,14 @@ public class LoginResource { @Context private ContainerRequestContext request; - + @GET public View getLoginPage(@QueryParam(RedirectingAuthFilter.REDIRECT_URI) URI redirectUri) { final String requestAuthority = request.getUriInfo().getBaseUri().getAuthority(); final String redirectAuthority = redirectUri.getAuthority(); - if (!requestAuthority.equals(redirectAuthority)){ - throw new BadRequestException(String.format( + if (!requestAuthority.equals(redirectAuthority)) { + throw new BadRequestException( + String.format( "The authorities of request uri (%s) and the redirect uri (%s) differ", requestAuthority, redirectAuthority)); diff --git a/backend/src/main/java/com/bakdata/conquery/resources/unprotected/TokenResource.java b/backend/src/main/java/com/bakdata/conquery/resources/unprotected/TokenResource.java index 8805f775108..27b53b16fae 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/unprotected/TokenResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/unprotected/TokenResource.java @@ -1,14 +1,15 @@ package com.bakdata.conquery.resources.unprotected; -import com.bakdata.conquery.apiv1.auth.JwtWrapper; -import com.bakdata.conquery.apiv1.auth.UsernamePasswordToken; -import com.bakdata.conquery.models.auth.basic.AccessTokenCreator; import jakarta.ws.rs.Consumes; import jakarta.ws.rs.NotAuthorizedException; import jakarta.ws.rs.POST; import jakarta.ws.rs.Path; import jakarta.ws.rs.Produces; import jakarta.ws.rs.core.MediaType; + +import com.bakdata.conquery.apiv1.auth.JwtWrapper; +import com.bakdata.conquery.apiv1.auth.UsernamePasswordToken; +import com.bakdata.conquery.models.auth.basic.AccessTokenCreator; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.shiro.authc.AuthenticationException; @@ -26,8 +27,7 @@ public class TokenResource { public JwtWrapper getToken(UsernamePasswordToken token) { try { return new JwtWrapper(realm.createAccessToken(token.getUser(), token.getPassword())); - } - catch (AuthenticationException e) { + } catch (AuthenticationException e) { log.warn("Failed to authorize request", e); throw new NotAuthorizedException("Failed to authenticate request. The cause has been logged."); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java index 2e6973f0e90..5b3e6f94a85 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java @@ -29,7 +29,12 @@ public class SqlExecutionManager extends ExecutionManager { private final SqlConverter converter; private final ConcurrentMap> runningExecutions; - public SqlExecutionManager(SqlConverter sqlConverter, SqlExecutionService sqlExecutionService, MetaStorage storage, DatasetRegistry datasetRegistry, ConqueryConfig config) { + public SqlExecutionManager( + SqlConverter sqlConverter, + SqlExecutionService sqlExecutionService, + MetaStorage storage, + DatasetRegistry datasetRegistry, + ConqueryConfig config) { super(storage, datasetRegistry, config); this.converter = sqlConverter; this.executionService = sqlExecutionService; @@ -49,12 +54,10 @@ protected void doExecute(E exec if (execution instanceof ManagedInternalForm managedForm) { CompletableFuture.allOf(managedForm.getSubQueries().values().stream().map(executionId -> { - addState(executionId, new SqlExecutionExecutionInfo()); - return executeAsync((ManagedQuery) executionId.resolve()); + addState(executionId, new SqlExecutionExecutionInfo()); + return executeAsync((ManagedQuery) executionId.resolve()); - }) - .toArray(CompletableFuture[]::new)) - .thenRun(() -> managedForm.finish(ExecutionState.DONE)); + }).toArray(CompletableFuture[]::new)).thenRun(() -> managedForm.finish(ExecutionState.DONE)); return; } @@ -64,25 +67,26 @@ protected void doExecute(E exec private CompletableFuture executeAsync(ManagedQuery managedQuery) { SqlQuery sqlQuery = converter.convert(managedQuery.getQuery()); - return CompletableFuture.supplyAsync(() -> executionService.execute(sqlQuery)) - .thenAccept(result -> { - ManagedExecutionId id = managedQuery.getId(); - - // We need to transfer the columns and data from the query result together with the execution lock to a new result - SqlExecutionExecutionInfo startResult = getExecutionInfo(id); - SqlExecutionExecutionInfo - finishResult = - new SqlExecutionExecutionInfo(ExecutionState.DONE, result.getColumnNames(), result.getTable(), result.getResultInfos(), startResult.getExecutingLock()); - addState(id, finishResult); - - managedQuery.finish(ExecutionState.DONE); - runningExecutions.remove(id); - }) - .exceptionally(e -> { - managedQuery.fail(ConqueryError.asConqueryError(e)); - runningExecutions.remove(managedQuery.getId()); - return null; - }); + return CompletableFuture.supplyAsync(() -> executionService.execute(sqlQuery)).thenAccept(result -> { + ManagedExecutionId id = managedQuery.getId(); + + // We need to transfer the columns and data from the query result together with the execution lock to a new result + SqlExecutionExecutionInfo startResult = getExecutionInfo(id); + SqlExecutionExecutionInfo finishResult = new SqlExecutionExecutionInfo( + ExecutionState.DONE, + result.getColumnNames(), + result.getTable(), + result.getResultInfos(), + startResult.getExecutingLock()); + addState(id, finishResult); + + managedQuery.finish(ExecutionState.DONE); + runningExecutions.remove(id); + }).exceptionally(e -> { + managedQuery.fail(ConqueryError.asConqueryError(e)); + runningExecutions.remove(managedQuery.getId()); + return null; + }); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlMatchingStats.java b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlMatchingStats.java index e35d7babd79..07eaa9487e4 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlMatchingStats.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlMatchingStats.java @@ -1,5 +1,11 @@ package com.bakdata.conquery.sql.conquery; +import static org.jooq.impl.DSL.*; + +import jakarta.validation.constraints.NotBlank; +import java.sql.Date; +import java.util.*; + import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; @@ -18,7 +24,6 @@ import com.google.common.collect.Sets; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; -import jakarta.validation.constraints.NotBlank; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; @@ -26,11 +31,6 @@ import org.jooq.Record; import org.jooq.exception.DataAccessException; -import java.sql.Date; -import java.util.*; - -import static org.jooq.impl.DSL.*; - @Slf4j @Data public class SqlMatchingStats { @@ -53,12 +53,15 @@ public class SqlMatchingStats { private final int matchingStatsWorkers; private final int matchingStatsRetries; - private static void assignStatsToPath(ConceptElement element, Map, MatchingStats.Entry> matchingStats, String entity, CDateRange span) { + private static void assignStatsToPath( + ConceptElement element, + Map, MatchingStats.Entry> matchingStats, + String entity, + CDateRange span) { while (element != null) { ConceptElementId id = element.getId(); - matchingStats.computeIfAbsent(id, (ignored) -> new MatchingStats.Entry()) - .addEvents(entity, 1, span); + matchingStats.computeIfAbsent(id, (ignored) -> new MatchingStats.Entry()).addEvents(entity, 1, span); element = element.getParent(); } } @@ -67,7 +70,11 @@ private static void assignStatsToPath(ConceptElement element, Map> collectReferencedFields(List conceptConditions) { - List> fields = conceptConditions.stream().flatMap(e -> e.conditions().keySet().stream()).distinct().toList(); + List> fields = conceptConditions.stream() + .flatMap( + e -> e.conditions().keySet().stream()) + .distinct() + .toList(); return fields; } @@ -150,7 +157,9 @@ private void assignStats(Map, MatchingStats.Entry> matchingS } @NotNull - private Map, MatchingStats.Entry> readStats(TreeConcept concept, SelectJoinStep selectJoinStep) { + private Map, MatchingStats.Entry> readStats( + TreeConcept concept, + SelectJoinStep selectJoinStep) { Map, MatchingStats.Entry> matchingStats = new HashMap<>(); Stopwatch stopwatch = Stopwatch.createStarted(); @@ -174,7 +183,9 @@ private Map, MatchingStats.Entry> readStats(TreeConcept conc Date min = record.get(LB_FIELD); Date max = record.get(UB_FIELD); - CDateRange span = CDateRange.of(min != null ? min.toLocalDate() : null, max != null ? max.toLocalDate() : null); + CDateRange span = CDateRange.of( + min != null ? min.toLocalDate() : null, + max != null ? max.toLocalDate() : null); assignStatsToPath(resolvedId, matchingStats, entity, span); } @@ -221,25 +232,31 @@ private List> createConceptIdsTable(Name tableName, List> keyF //TODO Option to create primaryKeys and indices here, but Hana is a bit flaky with it, would need to differentiate the impls. - CreateTableElementListStep createTable = - dslContext.createTable(tableName) - .columns(fields); + CreateTableElementListStep createTable = dslContext.createTable(tableName).columns(fields); createTable.execute(); return fields; } - public ListenableFuture collectMatchingStatsForConcept(TreeConcept concept, ListeningExecutorService executorService, int tries) { + public ListenableFuture collectMatchingStatsForConcept( + TreeConcept concept, + ListeningExecutorService executorService, + int tries) { return executorService.submit(() -> { dslContext.connection(cfg -> { try { SelectJoinStep matchingStatsStatement = createMatchingStatsStatement(concept); - Map, MatchingStats.Entry> matchingStats = readStats(concept, matchingStatsStatement); + Map, MatchingStats.Entry> matchingStats = readStats( + concept, + matchingStatsStatement); assignStats(matchingStats); } catch (DataAccessException e) { - log.debug("Failed to connect to database for concept {}. Retrying.", concept.getId(), (Exception) (log.isTraceEnabled() || tries == 0 ? e : null)); + log.debug( + "Failed to connect to database for concept {}. Retrying.", + concept.getId(), + (Exception) (log.isTraceEnabled() || tries == 0 ? e : null)); if (tries > 0) { collectMatchingStatsForConcept(concept, executorService, tries - 1); @@ -263,18 +280,24 @@ private SelectJoinStep createMatchingStatsStatement(TreeConcep Field[] validityDates = collectValidityDateFields(connector); - SelectConditionStep connectorTable = - dslContext.select( - TablePrimaryColumnUtil.findPrimaryColumn(connector.getResolvedTable(), defaultPrimaryColumn).as(PID_FIELD), - // The infinities are intentionally swapped - least(positiveInfinity, validityDates).as(LB_FIELD), - greatest(negativeInfinity, validityDates).as(UB_FIELD), - CONCEPT_ID_FIELD) - .from(table(name(connector.getResolvedTable().getName()))) - .leftJoin(idsTableName(concept.getName())) - // join onto the concept-ids table to assign the most specific id. - .on(getJoinConditions(concept, context)) - .where(connector.getCondition() != null ? connector.getCondition().convertToSqlCondition(context).condition() : noCondition()); + SelectConditionStep connectorTable = dslContext.select( + TablePrimaryColumnUtil.findPrimaryColumn(connector.getResolvedTable(), defaultPrimaryColumn) + .as( + PID_FIELD), + // The infinities are intentionally swapped + least(positiveInfinity, validityDates).as(LB_FIELD), + greatest(negativeInfinity, validityDates).as(UB_FIELD), + CONCEPT_ID_FIELD) + .from(table(name(connector.getResolvedTable().getName()))) + .leftJoin( + idsTableName(concept.getName())) + // join onto the concept-ids table to assign the most specific id. + .on(getJoinConditions(concept, context)) + .where( + connector.getCondition() != null ? connector.getCondition() + .convertToSqlCondition( + context) + .condition() : noCondition()); connectorTables.add(connectorTable); } @@ -282,9 +305,14 @@ private SelectJoinStep createMatchingStatsStatement(TreeConcep Name ct_name = name("connector_tables"); CommonTableExpression unioned = ct_name.as(unionSelects(connectorTables)); - SelectJoinStep> records = dslContext.with(unioned).select(unioned.field(CONCEPT_ID_FIELD), PID_FIELD, + SelectJoinStep> records = dslContext.with(unioned) + .select( + unioned.field(CONCEPT_ID_FIELD), + PID_FIELD, // The infinities are intentionally swapped - nullif(unioned.field(LB_FIELD), positiveInfinity).as(LB_FIELD), nullif(unioned.field(UB_FIELD), negativeInfinity).as(UB_FIELD)).from(ct_name); + nullif(unioned.field(LB_FIELD), positiveInfinity).as(LB_FIELD), + nullif(unioned.field(UB_FIELD), negativeInfinity).as(UB_FIELD)) + .from(ct_name); return records; } @@ -329,7 +357,9 @@ private Condition getJoinConditions(TreeConcept concept, CTConditionContext cont return reduced; } - private List expressionsToRows(List conceptConditions, List> allFields) { + private List expressionsToRows( + List conceptConditions, + List> allFields) { Map>, ConceptElement> byDepth = new HashMap<>(); for (CTCondition.ConceptConditions conceptCondition : conceptConditions) { @@ -354,7 +384,11 @@ private List expressionsToRows(List concept return elt; } if (prior.getDepth() == elt.getDepth() && !prior.equals(elt)) { - log.warn("Nodes {} and {} are mapped by the same params {}", prior.getId(), elt.getId(), params); + log.warn( + "Nodes {} and {} are mapped by the same params {}", + prior.getId(), + elt.getId(), + params); } return prior; }); @@ -378,13 +412,16 @@ private List expressionsToRows(List concept * Collect all mappings from values to conceptElement for the entire concept. This means the column-value and the auxiliary columns. * We use them to construct a table building an injective mapping from values to concept element that can be used for performant joins instead of resolving the concept every time. */ - private List collectAllExpressions(ConceptElement current, CTCondition.ConceptConditions parentConceptCondition, CTConditionContext context) { + private List collectAllExpressions( + ConceptElement current, + CTCondition.ConceptConditions parentConceptCondition, + CTConditionContext context) { final CTCondition.ConceptConditions forCurrent = switch (current) { case TreeConcept concept -> new CTCondition.ConceptConditions(concept, Collections.emptyMap()); // concept elements implicitly inherit the conditions of its parents case ConceptTreeChild child -> - child.getCondition().buildExpression(context, current).and(parentConceptCondition); + child.getCondition().buildExpression(context, current).and(parentConceptCondition); case null, default -> throw new IllegalStateException(); }; @@ -405,14 +442,21 @@ private List collectAllExpressions(ConceptElement * @param current * @param context TODO use this to implement joining in queries */ - private CTCondition.ConceptConditions collectExpressionsForSingleNode(ConceptElement current, CTConditionContext context) { + private CTCondition.ConceptConditions collectExpressionsForSingleNode( + ConceptElement current, + CTConditionContext context) { if (current instanceof TreeConcept concept) { return new CTCondition.ConceptConditions(concept, Collections.emptyMap()); } - CTCondition.ConceptConditions parentConceptCondition = collectExpressionsForSingleNode(current.getParent(), context); - CTCondition.ConceptConditions currentConceptCondition = ((ConceptTreeChild) current).getCondition().buildExpression(context, current); + CTCondition.ConceptConditions parentConceptCondition = collectExpressionsForSingleNode( + current.getParent(), + context); + CTCondition.ConceptConditions currentConceptCondition = ((ConceptTreeChild) current).getCondition() + .buildExpression( + context, + current); return currentConceptCondition.and(parentConceptCondition); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/Context.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/Context.java index 4bc8338bd1f..59936303386 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/Context.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/Context.java @@ -17,8 +17,8 @@ public interface Context { default NameGenerator getNameGenerator() { return getConversionContext().getNameGenerator(); } - - default SqlFunctionProvider getFunctionProvider(){ + + default SqlFunctionProvider getFunctionProvider() { return getConversionContext().getDialectBundle().getFunctionProvider(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/Conversions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/Conversions.java index 50116dea6e8..9185a669d32 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/Conversions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/Conversions.java @@ -29,8 +29,7 @@ public R convert(C node, X context) { if (maybeConverted.isPresent()) { if (converted == null) { converted = maybeConverted.get(); - } - else { + } else { throw new IllegalStateException("Multiple Converters for %s".formatted(node)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java index 742538f3a15..08f69cd8ef9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java @@ -1,5 +1,8 @@ package com.bakdata.conquery.sql.conversion; +import java.time.Clock; +import java.util.Locale; + import com.bakdata.conquery.apiv1.query.QueryDescription; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.IdColumnConfig; @@ -12,9 +15,6 @@ import lombok.NonNull; import org.jooq.DSLContext; -import java.time.Clock; -import java.util.Locale; - /** * Entry point for converting {@link QueryDescription} to an SQL query. */ @@ -29,12 +29,12 @@ public class NodeConversions extends Conversions { private static final String UNDERSCORE = "_"; private static final String WHITESPACE = " "; - private static QueryStep createExternalIdsCte(CQExternal external, SqlFunctionProvider functionProvider, ConversionContext context) { + private static QueryStep createExternalIdsCte( + CQExternal external, + SqlFunctionProvider functionProvider, + ConversionContext context) { List unions = new ArrayList<>(); for (Map.Entry entry : external.getValuesResolved().entrySet()) { List rowSelects = createRowSelects(entry, functionProvider); unions.addAll(rowSelects); } - Preconditions.checkArgument(!unions.isEmpty(), "Expecting at least 1 resolved row when converting a CQExternal"); - QueryStep allStep = QueryStep.createUnionAllStep(unions, CQ_EXTERNAL_IDS_CTE_NAME, Collections.emptyList(), context.isNegation(), functionProvider); + Preconditions.checkArgument( + !unions.isEmpty(), + "Expecting at least 1 resolved row when converting a CQExternal"); + QueryStep allStep = QueryStep.createUnionAllStep( + unions, + CQ_EXTERNAL_IDS_CTE_NAME, + Collections.emptyList(), + context.isNegation(), + functionProvider); Optional maybeValidityDate = allStep.getSelects().getValidityDate(); @@ -50,12 +60,18 @@ private static QueryStep createExternalIdsCte(CQExternal external, SqlFunctionPr } return QueryStep.builder() - .predecessors(List.of(allStep)) - .conditions(List.of(functionProvider.dateRestriction(functionProvider.forCDateRange(context.getDateRestrictionRange()), maybeValidityDate.get()))) - .selects(allStep.getQualifiedSelects()) - .fromTable(table(name(allStep.getCteName()))) - .cteName(CQ_EXTERNAL_IDS_CTE_NAME + "_date_restriction") - .build(); + .predecessors(List.of(allStep)) + .conditions( + List.of( + functionProvider.dateRestriction( + functionProvider.forCDateRange(context.getDateRestrictionRange()), + maybeValidityDate.get()))) + .selects(allStep.getQualifiedSelects()) + .fromTable( + table(name(allStep.getCteName()))) + .cteName( + CQ_EXTERNAL_IDS_CTE_NAME + "_date_restriction") + .build(); } @@ -63,16 +79,25 @@ private static QueryStep createExternalIdsCte(CQExternal external, SqlFunctionPr * For each entry, we need to create a SELECT statement of static values for each pid -> date set. For dialects that support date multiranges, * 1 row per ID is sufficient. For other dialects there can be multiple rows with the same pid -> date range from the date set. */ - private static List createRowSelects(Map.Entry entry, SqlFunctionProvider functionProvider) { + private static List createRowSelects( + Map.Entry entry, + SqlFunctionProvider functionProvider) { SqlIdColumns ids = createIdSelect(entry, functionProvider); - List validityDateEntries = functionProvider.forCDateSet(entry.getValue(), SharedAliases.DATES_COLUMN); + List validityDateEntries = functionProvider.forCDateSet( + entry.getValue(), + SharedAliases.DATES_COLUMN); return validityDateEntries.stream() - .map(validityDateEntry -> createIdRowSelect(ids, validityDateEntry, functionProvider)) - .toList(); + .map( + validityDateEntry -> createIdRowSelect(ids, validityDateEntry, functionProvider)) + .toList(); } - private static SqlIdColumns createIdSelect(Map.Entry entry, SqlFunctionProvider functionProvider) { - Field primaryColumn = functionProvider.externalId(entry.getKey()).as(SharedAliases.PRIMARY_COLUMN.getAlias()); + private static SqlIdColumns createIdSelect( + Map.Entry entry, + SqlFunctionProvider functionProvider) { + Field primaryColumn = functionProvider.externalId(entry.getKey()) + .as( + SharedAliases.PRIMARY_COLUMN.getAlias()); return new SqlIdColumns(primaryColumn); } @@ -80,21 +105,29 @@ private static SqlIdColumns createIdSelect(Map.Entry entry, Sq * For each entry, we need to create a SELECT statement of static values for each pid -> extras. */ private QueryStep createRowSelects( - Map.Entry entry, - List>> extra, - SqlFunctionProvider functionProvider + Map.Entry entry, + List>> extra, + SqlFunctionProvider functionProvider ) { SqlIdColumns ids = createIdSelect(entry); - List extraSelects = extra.stream().map(extraEntry -> createExtraColumnValue(extraEntry, functionProvider)).collect(Collectors.toList()); + List extraSelects = extra.stream() + .map( + extraEntry -> createExtraColumnValue(extraEntry, functionProvider)) + .collect(Collectors.toList()); return createExtraRowSelect(ids, extraSelects, functionProvider); } private static SqlIdColumns createIdSelect(Map.Entry entry) { - Field primaryColumn = DSL.val(entry.getKey()).coerce(String.class).as(SharedAliases.PRIMARY_COLUMN.getAlias()); + Field primaryColumn = DSL.val(entry.getKey()) + .coerce(String.class) + .as( + SharedAliases.PRIMARY_COLUMN.getAlias()); return new SqlIdColumns(primaryColumn); } - private static FieldWrapper createExtraColumnValue(Map.Entry> extraEntry, SqlFunctionProvider functionProvider) { + private static FieldWrapper createExtraColumnValue( + Map.Entry> extraEntry, + SqlFunctionProvider functionProvider) { Field extraValues = functionProvider.asArrayRepr(extraEntry.getValue()); final Name alias = name(extraEntry.getKey().replace(WHITESPACE, UNDERSCORE)); final Field withAlias = extraValues.as(alias); @@ -106,14 +139,11 @@ private static FieldWrapper createExtraColumnValue(Map.Entry{@code select 1 as "pid", '[2021-01-01,2022-01-01)'::daterange as "date_range"} */ private static QueryStep createIdRowSelect( - SqlIdColumns ids, - ColumnDateRange validityDate, - SqlFunctionProvider functionProvider + SqlIdColumns ids, + ColumnDateRange validityDate, + SqlFunctionProvider functionProvider ) { - Selects selects = Selects.builder() - .ids(ids) - .validityDate(Optional.ofNullable(validityDate)) - .build(); + Selects selects = Selects.builder().ids(ids).validityDate(Optional.ofNullable(validityDate)).build(); return wrapInQueryStep(selects, functionProvider); } @@ -122,14 +152,11 @@ private static QueryStep createIdRowSelect( * 
{@code select 1 as "pid", '1' as "payload_1"}
*/ private static QueryStep createExtraRowSelect( - SqlIdColumns ids, - List extraSelects, - SqlFunctionProvider functionProvider + SqlIdColumns ids, + List extraSelects, + SqlFunctionProvider functionProvider ) { - Selects selects = Selects.builder() - .ids(ids) - .sqlSelects(extraSelects) - .build(); + Selects selects = Selects.builder().ids(ids).sqlSelects(extraSelects).build(); return wrapInQueryStep(selects, functionProvider); } @@ -139,10 +166,7 @@ private static QueryStep wrapInQueryStep(Selects selects, SqlFunctionProvider fu // so we ensure there is some no-op table to select the static values from Table table = functionProvider.getNoOpTable(); - return QueryStep.builder() - .selects(selects) - .fromTable(table) - .build(); + return QueryStep.builder().selects(selects).fromTable(table).build(); } @Override @@ -165,16 +189,26 @@ public ConversionContext convert(CQExternal external, ConversionContext context) return withExternalIdCte.withExternalExtras(externalExtrasCte); } - private QueryStep createExternalExtrasCte(CQExternal external, SqlFunctionProvider functionProvider, ConversionContext context) { + private QueryStep createExternalExtrasCte( + CQExternal external, + SqlFunctionProvider functionProvider, + ConversionContext context) { List unions = new ArrayList<>(); for (Map.Entry entry : external.getValuesResolved().entrySet()) { List>> extrasForId = external.getExtrasForId(entry.getKey()); QueryStep rowSelects = createRowSelects(entry, extrasForId, functionProvider); unions.add(rowSelects); } - Preconditions.checkArgument(!unions.isEmpty(), "Expecting at least 1 converted resolved row when converting a CQExternal"); - return QueryStep.createUnionAllStep(unions, CQ_EXTERNAL_EXTRAS_CTE_NAME, Collections.emptyList(), context.isNegation(), functionProvider); + Preconditions.checkArgument( + !unions.isEmpty(), + "Expecting at least 1 converted resolved row when converting a CQExternal"); + return QueryStep.createUnionAllStep( + unions, + CQ_EXTERNAL_EXTRAS_CTE_NAME, + Collections.emptyList(), + context.isNegation(), + functionProvider); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java index 2bb90a6d62c..9226b7fae92 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java @@ -5,17 +5,18 @@ public class CQNegationConverter implements NodeConverter { - @Override - public Class getConversionClass() { - return CQNegation.class; - } + @Override + public Class getConversionClass() { + return CQNegation.class; + } - @Override - public ConversionContext convert(CQNegation negationNode, ConversionContext context) { - return context.getNodeConversions() - .convert(negationNode.getChild(), context.withNegation(true)) - .withNegation(false); - } + @Override + public ConversionContext convert(CQNegation negationNode, ConversionContext context) { + return context.getNodeConversions() + .convert(negationNode.getChild(), context.withNegation(true)) + .withNegation( + false); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java index d77d52a207d..951286eb597 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java @@ -19,15 +19,17 @@ public ConversionContext convert(CQOr orNode, ConversionContext context) { QueryStep joined; if (orNode.getChildren().size() == 1) { - ConversionContext withConvertedChild = context.getNodeConversions().convert(orNode.getChildren().get(0), context); + ConversionContext withConvertedChild = context.getNodeConversions() + .convert( + orNode.getChildren().get(0), + context); joined = withConvertedChild.getLastConvertedStep(); - } - else { + } else { joined = QueryStepJoiner.joinChildren( - orNode.getChildren(), - context, - ConqueryJoinType.OUTER_JOIN, - orNode.getDateAction() + orNode.getChildren(), + context, + ConqueryJoinType.OUTER_JOIN, + orNode.getDateAction() ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java index f45cba7d1bc..e4f8b4c2230 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java @@ -7,7 +7,6 @@ import com.bakdata.conquery.apiv1.query.CQYes; import com.bakdata.conquery.models.config.ColumnConfig; import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; @@ -30,16 +29,14 @@ public ConversionContext convert(CQYes cqYes, ConversionContext context) { Field primaryColumn = field(name(primaryColumnConfig.getField()), String.class); SqlIdColumns ids = new SqlIdColumns(primaryColumn); - Selects selects = Selects.builder().ids(ids) - .validityDate(Optional.of(context.getFunctionProvider().emptyColumnDateRange().asValidityDateRange(ALL_IDS_CTE))) - .build(); + Selects selects = Selects.builder() + .ids(ids) + .validityDate( + Optional.of(context.getFunctionProvider().emptyColumnDateRange().asValidityDateRange(ALL_IDS_CTE))) + .build(); org.jooq.Table fromTable = table(name(context.getIdColumns().getTable())); - QueryStep cqYesTep = QueryStep.builder() - .cteName(ALL_IDS_CTE) - .selects(selects) - .fromTable(fromTable) - .build(); + QueryStep cqYesTep = QueryStep.builder().cteName(ALL_IDS_CTE).selects(selects).fromTable(fromTable).build(); return context.withQueryStep(cqYesTep); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java index a2973ec90da..2eaa066325b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java @@ -13,7 +13,6 @@ import com.bakdata.conquery.sql.conversion.Context; import com.bakdata.conquery.sql.conversion.NodeConversions; import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; import com.bakdata.conquery.sql.conversion.model.NameGenerator; import com.bakdata.conquery.sql.conversion.model.QueryStep; @@ -25,7 +24,8 @@ import lombok.Value; import lombok.With; -//TODO this class is less context and more state. It's also incredibly dangerous because it hides a lot of moving parts and creates indirections. +// TODO this class is less context and more state. It's also incredibly dangerous because it hides a lot of moving parts +// and creates indirections. @Value @With @Builder(toBuilder = true) @@ -76,7 +76,6 @@ public boolean isWithStratification() { } - /** * Adds a query step to the list of {@link QueryStep} of this context. */ diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java index 8656cd9bf4a..b5b22494924 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.sql.conversion.cqelement.aggregation; -import static org.jooq.impl.DSL.field; -import static org.jooq.impl.DSL.inline; import java.sql.Date; import java.util.List; @@ -30,27 +28,32 @@ public class AnsiSqlDateAggregator implements SqlDateAggregator { @Override public QueryStep apply( - QueryStep joinedStep, - List carryThroughSelects, - DateAggregationDates dateAggregationDates, - DateAggregationAction dateAggregationAction, - ConversionContext conversionContext + QueryStep joinedStep, + List carryThroughSelects, + DateAggregationDates dateAggregationDates, + DateAggregationAction dateAggregationAction, + ConversionContext conversionContext ) { SqlAggregationAction aggregationAction = switch (dateAggregationAction) { case MERGE -> new MergeAggregateAction(joinedStep); case INTERSECT -> new IntersectAggregationAction(joinedStep); - default -> throw new IllegalStateException("Unexpected date aggregation action: %s".formatted(dateAggregationAction)); + default -> + throw new IllegalStateException( + "Unexpected date aggregation action: %s".formatted(dateAggregationAction)); }; - DateAggregationContext context = - DateAggregationContext.builder() - .sqlAggregationAction(aggregationAction) - .carryThroughSelects(carryThroughSelects) - .dateAggregationDates(dateAggregationDates) - .dateAggregationTables(aggregationAction.tableNames(conversionContext.getNameGenerator())) - .ids(joinedStep.getQualifiedSelects().getIds()) - .conversionContext(conversionContext) - .build(); + DateAggregationContext context = DateAggregationContext.builder() + .sqlAggregationAction( + aggregationAction) + .carryThroughSelects(carryThroughSelects) + .dateAggregationDates( + dateAggregationDates) + .dateAggregationTables( + aggregationAction.tableNames(conversionContext.getNameGenerator())) + .ids( + joinedStep.getQualifiedSelects().getIds()) + .conversionContext(conversionContext) + .build(); QueryStep finalDateAggregationStep = convertSteps(joinedStep, aggregationAction.dateAggregationCtes(), context); if (!aggregationAction.requiresIntervalPackingAfterwards()) { @@ -60,28 +63,32 @@ public QueryStep apply( Selects predecessorSelects = finalDateAggregationStep.getSelects(); SqlTables intervalPackingTables = IntervalPackingCteStep.createTables(finalDateAggregationStep, context); - IntervalPackingContext intervalPackingContext = - IntervalPackingContext.builder() - .ids(predecessorSelects.getIds()) - .daterange(predecessorSelects.getValidityDate().get()) - .predecessor(finalDateAggregationStep) - .carryThroughSelects(carryThroughSelects) - .tables(intervalPackingTables) - .conversionContext(conversionContext) - .build(); + IntervalPackingContext intervalPackingContext = IntervalPackingContext.builder() + .ids( + predecessorSelects.getIds()) + .daterange(predecessorSelects.getValidityDate().get()) + .predecessor( + finalDateAggregationStep) + .carryThroughSelects(carryThroughSelects) + .tables( + intervalPackingTables) + .conversionContext(conversionContext) + .build(); return this.intervalPacker.aggregateAsValidityDate(intervalPackingContext); } @Override - public ColumnDateRange getAggregatedValidityDate(DateAggregationDates dateAggregationDates, DateAggregationAction dateAggregationAction) { + public ColumnDateRange getAggregatedValidityDate( + DateAggregationDates dateAggregationDates, + DateAggregationAction dateAggregationAction) { //TODO(FK): i think this is only ever relevant with dateMode=Logical which i want to remove Field rangeStart = functionProvider.least(dateAggregationDates.allStarts()); Field rangeEnd = functionProvider.greatest(dateAggregationDates.allEnds()); return ColumnDateRange.of( - rangeStart.as(DateAggregationCte.RANGE_START), - rangeEnd.as(DateAggregationCte.RANGE_END) + rangeStart.as(DateAggregationCte.RANGE_START), + rangeEnd.as(DateAggregationCte.RANGE_END) ); } @@ -94,21 +101,28 @@ public QueryStep invertAggregatedIntervals(QueryStep baseStep, ConversionContext } Selects baseStepQualifiedSelects = baseStep.getQualifiedSelects(); - SqlTables dateAggregationTables = DateAggregationCteStep.createInvertTables(baseStep, conversionContext.getNameGenerator()); + SqlTables dateAggregationTables = DateAggregationCteStep.createInvertTables( + baseStep, + conversionContext.getNameGenerator()); DateAggregationContext context = DateAggregationContext.builder() - .sqlAggregationAction(null) // when inverting, an aggregation has already been applied - .carryThroughSelects(baseStepQualifiedSelects.getSqlSelects()) - .dateAggregationDates(dateAggregationDates) - .dateAggregationTables(dateAggregationTables) - .ids(baseStepQualifiedSelects.getIds()) - .conversionContext(conversionContext) - .build(); + .sqlAggregationAction(null) // when inverting, an aggregation has already been applied + .carryThroughSelects(baseStepQualifiedSelects.getSqlSelects()) + .dateAggregationDates( + dateAggregationDates) + .dateAggregationTables(dateAggregationTables) + .ids( + baseStepQualifiedSelects.getIds()) + .conversionContext(conversionContext) + .build(); return convertSteps(baseStep, DateAggregationCteStep.createInvertCtes(), context); } - private QueryStep convertSteps(QueryStep baseStep, List dateAggregationCTEs, DateAggregationContext context) { + private QueryStep convertSteps( + QueryStep baseStep, + List dateAggregationCTEs, + DateAggregationContext context) { QueryStep finalDateAggregationStep = baseStep; for (DateAggregationCte step : dateAggregationCTEs) { finalDateAggregationStep = step.convert(context, finalDateAggregationStep); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java index 7eae7cb5289..886826e3030 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java @@ -31,8 +31,7 @@ class DateAggregationContext implements Context, Qualifiable new ArrayList<>()) - .add(queryStep); + this.intervalMergeSteps.computeIfAbsent(dateAggregationCteStep, k -> new ArrayList<>()).add(queryStep); return this; } @@ -47,7 +46,7 @@ public QueryStep getStep(DateAggregationCteStep dateAggregationCteStep) { public List getSteps(DateAggregationCteStep dateAggregationCteStep) { if (dateAggregationCteStep != DateAggregationCteStep.NODE_NO_OVERLAP) { throw new UnsupportedOperationException( - "Only MergeCteStep.NODE_NO_OVERLAP has multiple steps. Use getStep() for all other DateAggregationSteps." + "Only MergeCteStep.NODE_NO_OVERLAP has multiple steps. Use getStep() for all other DateAggregationSteps." ); } return this.intervalMergeSteps.get(dateAggregationCteStep); @@ -56,10 +55,12 @@ public List getSteps(DateAggregationCteStep dateAggregationCteStep) { @Override public DateAggregationContext qualify(String qualifier) { return this.toBuilder() - .ids(this.ids.qualify(qualifier)) - .carryThroughSelects(QualifyingUtil.qualify(this.carryThroughSelects, qualifier)) - .dateAggregationDates(this.dateAggregationDates.qualify(qualifier)) - .build(); + .ids(this.ids.qualify(qualifier)) + .carryThroughSelects( + QualifyingUtil.qualify(this.carryThroughSelects, qualifier)) + .dateAggregationDates( + this.dateAggregationDates.qualify(qualifier)) + .build(); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCte.java index 721d99c84f2..555c42734d5 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCte.java @@ -25,8 +25,7 @@ public QueryStep convert(DateAggregationContext context, QueryStep previous) { QueryStep.QueryStepBuilder builder = this.convertStep(context, predecessor); if (cteStep != DateAggregationCteStep.NODE_NO_OVERLAP) { - builder = builder.cteName(dateAggregationTables.cteName(cteStep)) - .predecessors(List.of(previous)); + builder = builder.cteName(dateAggregationTables.cteName(cteStep)).predecessors(List.of(previous)); } if (cteStep != DateAggregationCteStep.INVERT && cteStep != DateAggregationCteStep.NODE_NO_OVERLAP) { builder = builder.fromTable(QueryStep.toTableLike(predecessor)); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCteStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCteStep.java index 0c156762f01..c7186d8e6ac 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCteStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCteStep.java @@ -17,31 +17,33 @@ enum DateAggregationCteStep implements CteStep { // merge or intersect - OVERLAP("overlap", OverlapCte::new, null), - INTERMEDIATE_TABLE("no_overlap", IntermediateTableCte::new, null), - NODE_NO_OVERLAP("node_no_overlap", NodeNoOverlapCte::new, INTERMEDIATE_TABLE), - MERGE("merge", MergeCte::new, OVERLAP), + OVERLAP("overlap", OverlapCte::new, null), INTERMEDIATE_TABLE( + "no_overlap", + IntermediateTableCte::new, + null), NODE_NO_OVERLAP("node_no_overlap", NodeNoOverlapCte::new, INTERMEDIATE_TABLE), MERGE( + "merge", + MergeCte::new, + OVERLAP), // invert - ROW_NUMBER("row_numbers", RowNumberCte::new, null), - INVERT("inverted_dates", InvertCte::new, ROW_NUMBER); + ROW_NUMBER("row_numbers", RowNumberCte::new, null), INVERT("inverted_dates", InvertCte::new, ROW_NUMBER); private static final List MERGE_STEPS = List.of( - OVERLAP, - INTERMEDIATE_TABLE, - NODE_NO_OVERLAP, - MERGE + OVERLAP, + INTERMEDIATE_TABLE, + NODE_NO_OVERLAP, + MERGE ); private static final List INTERSECT_STEPS = List.of( - OVERLAP, - INTERMEDIATE_TABLE, - MERGE + OVERLAP, + INTERMEDIATE_TABLE, + MERGE ); private static final List INVERT_STEPS = List.of( - ROW_NUMBER, - INVERT + ROW_NUMBER, + INVERT ); private final String suffix; @@ -73,12 +75,13 @@ public static SqlTables createInvertTables(QueryStep joinedTable, NameGenerator } private static List createCtes(List requiredSteps) { - return requiredSteps.stream() - .map(cteStep -> cteStep.getStepConstructor().create(cteStep)) - .toList(); + return requiredSteps.stream().map(cteStep -> cteStep.getStepConstructor().create(cteStep)).toList(); } - private static SqlTables createTables(List requiredSteps, QueryStep joinedTable, NameGenerator nameGenerator) { + private static SqlTables createTables( + List requiredSteps, + QueryStep joinedTable, + NameGenerator nameGenerator) { Set asSet = new HashSet<>(requiredSteps); Map cteNameMap = CteStep.createCteNameMap(asSet, joinedTable.getCteName(), nameGenerator); Map predecessorMap = CteStep.getDefaultPredecessorMap(asSet); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java index c3c7e708e6f..638368b05b1 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationDates.java @@ -4,7 +4,6 @@ import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; @@ -27,24 +26,27 @@ public class DateAggregationDates { public static DateAggregationDates forValidityDates(final List> validityDates) { final List filtered = validityDates.stream() - .filter(Optional::isPresent) - .map(Optional::get) - .toList(); + .filter(Optional::isPresent) + .map( + Optional::get) + .toList(); return new DateAggregationDates(filtered); } public static DateAggregationDates forSingleStep(QueryStep queryStep) { List validityDates = queryStep.getSelects() - .getValidityDate() - .map(List::of) - .orElse(Collections.emptyList()); + .getValidityDate() + .map(List::of) + .orElse( + Collections.emptyList()); return new DateAggregationDates(validityDates); } public static DateAggregationDates forSteps(List querySteps) { final List validityDates = querySteps.stream() - .flatMap(queryStep -> queryStep.getQualifiedSelects().getValidityDate().stream()) - .toList(); + .flatMap( + queryStep -> queryStep.getQualifiedSelects().getValidityDate().stream()) + .toList(); return new DateAggregationDates(validityDates); } @@ -62,15 +64,17 @@ public List> allEnds() { public List allStartsAndEnds() { return this.validityDates.stream() - .flatMap(validityDate -> validityDate.toFields().stream()) - .map(FieldWrapper::new) - .collect(Collectors.toList()); + .flatMap(validityDate -> validityDate.toFields().stream()) + .map( + FieldWrapper::new) + .collect(Collectors.toList()); } public DateAggregationDates qualify(String qualifier) { List qualified = this.validityDates.stream() - .map(validityDate -> validityDate.qualify(qualifier)) - .toList(); + .map( + validityDate -> validityDate.qualify(qualifier)) + .toList(); // validity dates will already be numerated, no we don't need no apply a counter again return new DateAggregationDates(qualified); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java index d7ccea99661..27ec654dfb1 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java @@ -25,14 +25,12 @@ public IntermediateTableCte(DateAggregationCteStep cteStep) { protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, String predecessor) { - List intermediateTableSelects = context.getSqlAggregationAction().getIntermediateTableSelects( + List intermediateTableSelects = context.getSqlAggregationAction() + .getIntermediateTableSelects( context.getDateAggregationDates(), context.getCarryThroughSelects() - ); - Selects selects = Selects.builder() - .ids(context.getIds()) - .sqlSelects(intermediateTableSelects) - .build(); + ); + Selects selects = Selects.builder().ids(context.getIds()).sqlSelects(intermediateTableSelects).build(); DateAggregationDates dateAggregationDates = context.getDateAggregationDates(); List> allStarts = dateAggregationDates.allStarts(); @@ -41,16 +39,11 @@ protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, SqlFunctionProvider functionProvider = context.getFunctionProvider(); Condition startBeforeEnd = functionProvider.greatest(allStarts).lessThan(functionProvider.least(allEnds)); - Condition startIsNull = allStarts.stream() - .map(Field::isNull) - .reduce(Condition::or) - .orElseThrow(); + Condition startIsNull = allStarts.stream().map(Field::isNull).reduce(Condition::or).orElseThrow(); Condition intermediateTableCondition = startIsNull.orNot(startBeforeEnd); - return QueryStep.builder() - .selects(selects) - .conditions(List.of(intermediateTableCondition)); + return QueryStep.builder().selects(selects).conditions(List.of(intermediateTableCondition)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java index 70ee2023fb4..a316271049c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntersectAggregationAction.java @@ -33,31 +33,33 @@ public List dateAggregationCtes() { } @Override - public ColumnDateRange getOverlapValidityDate(DateAggregationDates dateAggregationDates, SqlFunctionProvider functionProvider) { + public ColumnDateRange getOverlapValidityDate( + DateAggregationDates dateAggregationDates, + SqlFunctionProvider functionProvider) { Field rangeStart = functionProvider.greatest(dateAggregationDates.allStarts()); Field rangeEnd = functionProvider.least(dateAggregationDates.allEnds()); return ColumnDateRange.of( - rangeStart.as(DateAggregationCte.RANGE_START), - rangeEnd.as(DateAggregationCte.RANGE_END) + rangeStart.as(DateAggregationCte.RANGE_START), + rangeEnd.as(DateAggregationCte.RANGE_END) ); } @Override - public List getIntermediateTableSelects(DateAggregationDates dateAggregationDates, List carryThroughSelects) { + public List getIntermediateTableSelects( + DateAggregationDates dateAggregationDates, + List carryThroughSelects) { - List> nulledRangeStartAndEnd = - Stream.of( - DSL.inline(null, Date.class).as(DateAggregationCte.RANGE_START), - DSL.inline(null, Date.class).as(DateAggregationCte.RANGE_END) - ) - .map(FieldWrapper::new) - .collect(Collectors.toList()); + List> nulledRangeStartAndEnd = Stream.of( + DSL.inline(null, Date.class).as(DateAggregationCte.RANGE_START), + DSL.inline(null, Date.class).as(DateAggregationCte.RANGE_END) + ).map(FieldWrapper::new).collect(Collectors.toList()); return Stream.of(nulledRangeStartAndEnd, carryThroughSelects) - .flatMap(Collection::stream) - .collect(Collectors.toList()); + .flatMap(Collection::stream) + .collect( + Collectors.toList()); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java index abd998364e2..263d102691f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java @@ -20,7 +20,6 @@ import org.jooq.Record; import org.jooq.TableLike; import org.jooq.TableOnConditionStep; -import org.jooq.impl.DSL; /** * Concept for date range inversion taken from Inverting date ranges. @@ -50,49 +49,57 @@ protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, Selects invertSelects = getInvertSelects(rowNumberStep, coalescedIds, context); TableOnConditionStep fromTable = selfJoinWithShiftedRows(leftIds, rightIds, rowNumberStep); - return QueryStep.builder() - .selects(invertSelects) - .fromTable(fromTable); + return QueryStep.builder().selects(invertSelects).fromTable(fromTable); } - private Selects getInvertSelects(QueryStep rowNumberStep, SqlIdColumns coalescedIds, DateAggregationContext context) { + private Selects getInvertSelects( + QueryStep rowNumberStep, + SqlIdColumns coalescedIds, + DateAggregationContext context) { SqlFunctionProvider functionProvider = context.getFunctionProvider(); ColumnDateRange validityDate = rowNumberStep.getSelects().getValidityDate().get(); Field rangeStart = coalesce( - QualifyingUtil.qualify(validityDate.getEnd(), ROWS_LEFT_TABLE_NAME), - functionProvider.getMinDateExpression() + QualifyingUtil.qualify(validityDate.getEnd(), ROWS_LEFT_TABLE_NAME), + functionProvider.getMinDateExpression() ).as(DateAggregationCte.RANGE_START); Field rangeEnd = coalesce( - QualifyingUtil.qualify(validityDate.getStart(), ROWS_RIGHT_TABLE_NAME), - functionProvider.getMaxDateExpression() + QualifyingUtil.qualify(validityDate.getStart(), ROWS_RIGHT_TABLE_NAME), + functionProvider.getMaxDateExpression() ).as(DateAggregationCte.RANGE_END); return Selects.builder() - .ids(coalescedIds) - .validityDate(Optional.of(ColumnDateRange.of(rangeStart, rangeEnd))) - .sqlSelects(context.getCarryThroughSelects()) - .build(); + .ids(coalescedIds) + .validityDate( + Optional.of(ColumnDateRange.of(rangeStart, rangeEnd))) + .sqlSelects(context.getCarryThroughSelects()) + .build(); } - private TableOnConditionStep selfJoinWithShiftedRows(SqlIdColumns leftIds, SqlIdColumns rightIds, QueryStep rowNumberStep) { + private TableOnConditionStep selfJoinWithShiftedRows( + SqlIdColumns leftIds, + SqlIdColumns rightIds, + QueryStep rowNumberStep) { - Field leftRowNumber = field(name(ROWS_LEFT_TABLE_NAME, RowNumberCte.ROW_NUMBER_FIELD_NAME), Integer.class) - .plus(1); - Field rightRowNumber = field(name(ROWS_RIGHT_TABLE_NAME, RowNumberCte.ROW_NUMBER_FIELD_NAME), Integer.class); + Field leftRowNumber = field( + name(ROWS_LEFT_TABLE_NAME, RowNumberCte.ROW_NUMBER_FIELD_NAME), + Integer.class).plus(1); + Field rightRowNumber = field( + name(ROWS_RIGHT_TABLE_NAME, RowNumberCte.ROW_NUMBER_FIELD_NAME), + Integer.class); Condition[] joinConditions = Stream.concat( - Stream.of(leftRowNumber.eq(rightRowNumber)), - leftIds.join(rightIds).stream() - ) - .toArray(Condition[]::new); + Stream.of(leftRowNumber.eq(rightRowNumber)), + leftIds.join(rightIds).stream() + ).toArray(Condition[]::new); TableLike rowNumberTable = QueryStep.toTableLike(rowNumberStep.getCteName()); return rowNumberTable.asTable(ROWS_LEFT_TABLE_NAME) - .fullJoin(rowNumberTable.asTable(ROWS_RIGHT_TABLE_NAME)) - .on(joinConditions); + .fullJoin(rowNumberTable.asTable(ROWS_RIGHT_TABLE_NAME)) + .on( + joinConditions); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java index d469015d413..7fcdd347053 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeAggregateAction.java @@ -31,22 +31,27 @@ public List dateAggregationCtes() { } @Override - public ColumnDateRange getOverlapValidityDate(DateAggregationDates dateAggregationDates, SqlFunctionProvider functionProvider) { + public ColumnDateRange getOverlapValidityDate( + DateAggregationDates dateAggregationDates, + SqlFunctionProvider functionProvider) { Field rangeStart = functionProvider.least(dateAggregationDates.allStarts()); Field rangeEnd = functionProvider.greatest(dateAggregationDates.allEnds()); return ColumnDateRange.of( - rangeStart.as(DateAggregationCte.RANGE_START), - rangeEnd.as(DateAggregationCte.RANGE_END) + rangeStart.as(DateAggregationCte.RANGE_START), + rangeEnd.as(DateAggregationCte.RANGE_END) ); } @Override - public List getIntermediateTableSelects(DateAggregationDates dateAggregationDates, List carryThroughSelects) { + public List getIntermediateTableSelects( + DateAggregationDates dateAggregationDates, + List carryThroughSelects) { return Stream.of(dateAggregationDates.allStartsAndEnds(), carryThroughSelects) - .flatMap(Collection::stream) - .collect(Collectors.toList()); + .flatMap( + Collection::stream) + .collect(Collectors.toList()); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeCte.java index 4fd1bc36cfa..ce8268884a9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/MergeCte.java @@ -22,18 +22,20 @@ protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, List noOverlapSteps = aggregationAction.getNoOverlapSelects(context); QueryStep overlapStep = aggregationAction.getOverlapStep(context); - List unionSteps = noOverlapSteps.stream().map(MergeCte::createUnionStep).collect(Collectors.toList()); + List unionSteps = noOverlapSteps.stream() + .map(MergeCte::createUnionStep) + .collect( + Collectors.toList()); - return QueryStep.builder() - .selects(overlapStep.getQualifiedSelects()) - .union(unionSteps); + return QueryStep.builder().selects(overlapStep.getQualifiedSelects()).union(unionSteps); } private static QueryStep createUnionStep(QueryStep noOverlapStep) { return QueryStep.builder() - .selects(noOverlapStep.getQualifiedSelects()) - .fromTable(QueryStep.toTableLike(noOverlapStep.getCteName())) - .build(); + .selects(noOverlapStep.getQualifiedSelects()) + .fromTable( + QueryStep.toTableLike(noOverlapStep.getCteName())) + .build(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java index 0da77091486..072a0b8cfd5 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java @@ -32,7 +32,10 @@ protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, QueryStep intermediateTableStep = context.getStep(DateAggregationCteStep.INTERMEDIATE_TABLE); // first no-overlap step has intermediate table as predecessor - QueryStep.QueryStepBuilder noOverlapStep = createNoOverlapStep(validityDates.next(), context, intermediateTableStep); + QueryStep.QueryStepBuilder noOverlapStep = createNoOverlapStep( + validityDates.next(), + context, + intermediateTableStep); // each following step has it's predeceasing no-overlap as predecessor while (validityDates.hasNext()) { @@ -46,9 +49,9 @@ protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, } private QueryStep.QueryStepBuilder createNoOverlapStep( - ColumnDateRange validityDate, - DateAggregationContext context, - QueryStep predecessor + ColumnDateRange validityDate, + DateAggregationContext context, + QueryStep predecessor ) { SqlTables dateAggregationTables = context.getDateAggregationTables(); @@ -60,19 +63,24 @@ private QueryStep.QueryStepBuilder createNoOverlapStep( Field asRangeStart = start.as(DateAggregationCte.RANGE_START); String intermediateTableCteName = dateAggregationTables.getPredecessor(getCteStep()); Selects nodeNoOverlapSelects = Selects.builder() - .ids(context.getIds()) - .validityDate(Optional.of(ColumnDateRange.of(asRangeStart, asRangeEnd))) - .sqlSelects(context.getCarryThroughSelects()) - .build(); + .ids(context.getIds()) + .validityDate( + Optional.of(ColumnDateRange.of(asRangeStart, asRangeEnd))) + .sqlSelects( + context.getCarryThroughSelects()) + .build(); Condition startNotNull = start.isNotNull(); return QueryStep.builder() - .cteName("%s_%s".formatted(dateAggregationTables.cteName(DateAggregationCteStep.NODE_NO_OVERLAP), counter)) - .selects(nodeNoOverlapSelects) - .fromTable(QueryStep.toTableLike(intermediateTableCteName)) - .conditions(List.of(startNotNull)) - .predecessors(List.of(predecessor)); + .cteName( + "%s_%s".formatted(dateAggregationTables.cteName(DateAggregationCteStep.NODE_NO_OVERLAP), counter)) + .selects( + nodeNoOverlapSelects) + .fromTable(QueryStep.toTableLike(intermediateTableCteName)) + .conditions( + List.of(startNotNull)) + .predecessors(List.of(predecessor)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java index 1d6790ea23b..a6d649ad0fa 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java @@ -30,23 +30,23 @@ protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, List> allStarts = dateAggregationDates.allStarts(); List> allEnds = dateAggregationDates.allEnds(); - ColumnDateRange overlapValidityDate = context.getSqlAggregationAction().getOverlapValidityDate(context.getDateAggregationDates(), functionProvider); + ColumnDateRange overlapValidityDate = context.getSqlAggregationAction() + .getOverlapValidityDate( + context.getDateAggregationDates(), + functionProvider); Selects overlapSelects = Selects.builder() - .ids(context.getIds()) - .validityDate(Optional.of(overlapValidityDate.asValidityDateRange(predecessor))) - .sqlSelects(context.getCarryThroughSelects()) - .build(); + .ids(context.getIds()) + .validityDate( + Optional.of(overlapValidityDate.asValidityDateRange(predecessor))) + .sqlSelects( + context.getCarryThroughSelects()) + .build(); Condition startBeforeEnd = functionProvider.greatest(allStarts).lessThan(functionProvider.least(allEnds)); - Condition allStartsNotNull = allStarts.stream() - .map(Field::isNotNull) - .reduce(Condition::and) - .orElseThrow(); + Condition allStartsNotNull = allStarts.stream().map(Field::isNotNull).reduce(Condition::and).orElseThrow(); Condition overlapConditions = allStartsNotNull.and(startBeforeEnd); - return QueryStep.builder() - .selects(overlapSelects) - .conditions(List.of(overlapConditions)); + return QueryStep.builder().selects(overlapSelects).conditions(List.of(overlapConditions)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java index 01c1dd5299b..4b75d0ed15a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java @@ -32,20 +32,22 @@ protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context, SqlIdColumns ids = context.getIds(); ColumnDateRange aggregatedValidityDate = context.getDateAggregationDates().getValidityDates().get(0); - Field rowNumber = DSL.rowNumber().over(DSL.partitionBy(ids.toFields()).orderBy(aggregatedValidityDate.getStart())) - .as(ROW_NUMBER_FIELD_NAME); + Field rowNumber = DSL.rowNumber() + .over( + DSL.partitionBy(ids.toFields()).orderBy(aggregatedValidityDate.getStart())) + .as(ROW_NUMBER_FIELD_NAME); ArrayList selects = new ArrayList<>(context.getCarryThroughSelects()); selects.add(new FieldWrapper<>(rowNumber)); Selects rowNumberSelects = Selects.builder() - .ids(ids) - .validityDate(Optional.of(aggregatedValidityDate)) - .sqlSelects(selects) - .build(); + .ids(ids) + .validityDate( + Optional.of(aggregatedValidityDate)) + .sqlSelects(selects) + .build(); - return QueryStep.builder() - .selects(rowNumberSelects); + return QueryStep.builder().selects(rowNumberSelects); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java index 5b98c52122c..937395981bf 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/SqlAggregationAction.java @@ -19,9 +19,13 @@ interface SqlAggregationAction { List dateAggregationCtes(); - ColumnDateRange getOverlapValidityDate(DateAggregationDates dateAggregationDates, SqlFunctionProvider functionProvider); + ColumnDateRange getOverlapValidityDate( + DateAggregationDates dateAggregationDates, + SqlFunctionProvider functionProvider); - List getIntermediateTableSelects(DateAggregationDates dateAggregationDates, List carryThroughSelects); + List getIntermediateTableSelects( + DateAggregationDates dateAggregationDates, + List carryThroughSelects); List getNoOverlapSelects(DateAggregationContext dateAggregationContext); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java index 85179b70430..437c5eeb421 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java @@ -21,38 +21,42 @@ public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { Selects aggregationFilterSelects = collectSelects(tableContext); - List aggregationFilterConditions = tableContext.getSqlFilters().stream() - .flatMap(conceptFilter -> conceptFilter.getWhereClauses().getGroupFilters().stream()) - .map(WhereCondition::condition) - .toList(); - - return QueryStep.builder() - .selects(aggregationFilterSelects) - .conditions(aggregationFilterConditions); + List aggregationFilterConditions = tableContext.getSqlFilters() + .stream() + .flatMap( + conceptFilter -> conceptFilter.getWhereClauses().getGroupFilters().stream()) + .map( + WhereCondition::condition) + .toList(); + + return QueryStep.builder().selects(aggregationFilterSelects).conditions(aggregationFilterConditions); } private Selects collectSelects(CQTableContext tableContext) { QueryStep previous = tableContext.getPrevious(); Selects previousSelects = previous.getQualifiedSelects(); - List forAggregationFilterStep = - tableContext.allSqlSelects().stream() - .flatMap(sqlSelects -> sqlSelects.getFinalSelects().stream()) - .map(sqlSelect -> { - // universal selects like an ExistsSelect have no predecessor in preceding CTE - if (sqlSelect.isUniversal()) { - return sqlSelect; - } - return sqlSelect.qualify(previous.getCteName()); - }) - .collect(Collectors.toList()); + List forAggregationFilterStep = tableContext.allSqlSelects() + .stream() + .flatMap( + sqlSelects -> sqlSelects.getFinalSelects().stream()) + .map(sqlSelect -> { + // universal selects like an ExistsSelect have no predecessor in preceding CTE + if (sqlSelect.isUniversal()) { + return sqlSelect; + } + return sqlSelect.qualify(previous.getCteName()); + }) + .collect(Collectors.toList()); return Selects.builder() - .ids(previousSelects.getIds()) - .stratificationDate(previousSelects.getStratificationDate()) - .validityDate(previousSelects.getValidityDate()) - .sqlSelects(forAggregationFilterStep) - .build(); + .ids(previousSelects.getIds()) + .stratificationDate( + previousSelects.getStratificationDate()) + .validityDate(previousSelects.getValidityDate()) + .sqlSelects( + forAggregationFilterStep) + .build(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java index 26ae3c67728..b86bc0e5f1f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java @@ -16,28 +16,28 @@ class AggregationSelectCte extends ConnectorCte { @Override public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { - List requiredInAggregationFilterStep = tableContext.allSqlSelects().stream() - .flatMap(sqlSelects -> sqlSelects.getAggregationSelects().stream()) - .toList(); + List requiredInAggregationFilterStep = tableContext.allSqlSelects() + .stream() + .flatMap( + sqlSelects -> sqlSelects.getAggregationSelects().stream()) + .toList(); Selects predecessorSelects = tableContext.getPrevious().getQualifiedSelects(); SqlIdColumns ids = predecessorSelects.getIds(); Optional stratificationDate = predecessorSelects.getStratificationDate(); Selects aggregationSelectSelects = Selects.builder() - .ids(ids) - .stratificationDate(stratificationDate) - .sqlSelects(requiredInAggregationFilterStep) - .build(); + .ids(ids) + .stratificationDate(stratificationDate) + .sqlSelects( + requiredInAggregationFilterStep) + .build(); List> groupByFields = Stream.concat( - ids.toFields().stream(), - stratificationDate.stream().flatMap(range -> range.toFields().stream()) - ) - .toList(); - - return QueryStep.builder() - .selects(aggregationSelectSelects) - .groupBy(groupByFields); + ids.toFields().stream(), + stratificationDate.stream().flatMap(range -> range.toFields().stream()) + ).toList(); + + return QueryStep.builder().selects(aggregationSelectSelects).groupBy(groupByFields); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index a3ecc000fdf..3aa16a81a90 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; +import static org.jooq.impl.DSL.*; + import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -38,22 +40,24 @@ import org.jooq.Record; import org.jooq.TableLike; -import static org.jooq.impl.DSL.*; - public class CQConceptConverter implements NodeConverter { private final List connectorCTEs; public CQConceptConverter() { this.connectorCTEs = List.of( - new PreprocessingCte(), - new AggregationSelectCte(), - new JoinBranchesCte(), - new AggregationFilterCte() + new PreprocessingCte(), + new AggregationSelectCte(), + new JoinBranchesCte(), + new AggregationFilterCte() ); } - private static QueryStep finishConceptConversion(QueryStep predecessor, CQConcept cqConcept, TablePath tablePath, ConversionContext context) { + private static QueryStep finishConceptConversion( + QueryStep predecessor, + CQConcept cqConcept, + TablePath tablePath, + ConversionContext context) { ConceptSqlTables universalTables = tablePath.createConceptTables(predecessor); @@ -61,67 +65,87 @@ private static QueryStep finishConceptConversion(QueryStep predecessor, CQConcep Optional validityDate = predecessorSelects.getValidityDate(); SqlIdColumns ids = predecessorSelects.getIds(); - SelectContext selectContext = SelectContext.create(ids, validityDate, universalTables, context); - List converted = cqConcept.getSelects().stream() - .map(selectId -> { - Select select = selectId.resolve(); - return context.getDialectBundle().getSelectConverter(select).conceptSelect(select, selectContext); - }) - .toList(); + SelectContext selectContext = SelectContext.create( + ids, + validityDate, + universalTables, + context); + List converted = cqConcept.getSelects().stream().map(selectId -> { + Select select = selectId.resolve(); + return context.getDialectBundle().getSelectConverter(select).conceptSelect(select, selectContext); + }).toList(); List queriesToJoin = new ArrayList<>(); queriesToJoin.add(predecessor); - converted.stream().map(ConceptSqlSelects::getAdditionalPredecessor).filter(Optional::isPresent).map(Optional::get).forEach(queriesToJoin::add); + converted.stream() + .map(ConceptSqlSelects::getAdditionalPredecessor) + .filter(Optional::isPresent) + .map( + Optional::get) + .forEach(queriesToJoin::add); if (universalTables.isRequiredStep(ConceptCteStep.INTERVAL_PACKING_SELECTS)) { - QueryStep eventDateSelectsStep = IntervalPackingSelectsCte.forConcept(predecessor, universalTables, converted, context); + QueryStep eventDateSelectsStep = IntervalPackingSelectsCte.forConcept( + predecessor, + universalTables, + converted, + context); queriesToJoin.add(eventDateSelectsStep); } // combine all universal selects and connector selects from preceding step List allConceptSelects = Stream.concat( - converted.stream().flatMap(sqlSelects -> sqlSelects.getFinalSelects().stream()), - // aggregate special selects (e.g. Exists) - predecessor.getQualifiedSelects().getSqlSelects().stream().map(SqlSelect::connectorAggregate) - ) - .toList(); + converted.stream().flatMap(sqlSelects -> sqlSelects.getFinalSelects().stream()), + // aggregate special selects (e.g. Exists) + predecessor.getQualifiedSelects().getSqlSelects().stream().map(SqlSelect::connectorAggregate) + ).toList(); Selects finalSelects = Selects.builder() - .ids(ids) - .stratificationDate(predecessorSelects.getStratificationDate()) - .validityDate(validityDate) - .sqlSelects(allConceptSelects) - .build(); - - TableLike joinedTable = QueryStepJoiner.constructJoinedTable(queriesToJoin, ConqueryJoinType.INNER_JOIN, context); + .ids(ids) + .stratificationDate( + predecessorSelects.getStratificationDate()) + .validityDate(validityDate) + .sqlSelects( + allConceptSelects) + .build(); + + TableLike joinedTable = QueryStepJoiner.constructJoinedTable( + queriesToJoin, + ConqueryJoinType.INNER_JOIN, + context); // group by everything which is not part of an aggregation in this step - List> groupByFields = - Stream.concat( - finalSelects.nonExplicitSelects().stream(), - finalSelects.getSqlSelects().stream() - .filter(Predicate.not(SqlSelect::isUniversal)) - .flatMap(sqlSelect -> sqlSelect.toFields().stream()) - ).toList(); + List> groupByFields = Stream.concat( + finalSelects.nonExplicitSelects().stream(), + finalSelects.getSqlSelects() + .stream() + .filter(Predicate.not(SqlSelect::isUniversal)) + .flatMap( + sqlSelect -> sqlSelect.toFields().stream()) + ).toList(); return QueryStep.builder() - .cteName(universalTables.cteName(ConceptCteStep.UNIVERSAL_SELECTS)) - .selects(finalSelects) - .fromTable(joinedTable) - .groupBy(groupByFields) - .predecessors(queriesToJoin) - .negate(context.isNegation()) - .build(); + .cteName(universalTables.cteName(ConceptCteStep.UNIVERSAL_SELECTS)) + .selects( + finalSelects) + .fromTable(joinedTable) + .groupBy(groupByFields) + .predecessors(queriesToJoin) + .negate( + context.isNegation()) + .build(); } public static SqlIdColumns convertIds(CQConcept cqConcept, CQTable cqTable, ConversionContext conversionContext) { Table table = cqTable.getConnector().resolve().getResolvedTable(); - Field primaryColumn = TablePrimaryColumnUtil.findPrimaryColumn(table, conversionContext.getDefaultPrimaryColumn()); + Field primaryColumn = TablePrimaryColumnUtil.findPrimaryColumn( + table, + conversionContext.getDefaultPrimaryColumn()); - if (cqConcept.isExcludeFromSecondaryId() - || conversionContext.getSecondaryIdDescription() == null - || !cqTable.hasSelectedSecondaryId(conversionContext.getSecondaryIdDescription().getId()) + if (cqConcept.isExcludeFromSecondaryId() || conversionContext.getSecondaryIdDescription() == null || !cqTable + .hasSelectedSecondaryId( + conversionContext.getSecondaryIdDescription().getId()) ) { return new SqlIdColumns(primaryColumn).withAlias(); } @@ -129,11 +153,11 @@ public static SqlIdColumns convertIds(CQConcept cqConcept, CQTable cqTable, Conv Column secondaryIdColumn = table.findSecondaryIdColumn(conversionContext.getSecondaryIdDescription().getId()); Preconditions.checkArgument( - secondaryIdColumn != null, - "Expecting Table %s to have a matching secondary id for %s".formatted( - table, - conversionContext.getSecondaryIdDescription() - ) + secondaryIdColumn != null, + "Expecting Table %s to have a matching secondary id for %s".formatted( + table, + conversionContext.getSecondaryIdDescription() + ) ); Field secondaryId = field(name(table.getName(), secondaryIdColumn.getName()), String.class); @@ -165,7 +189,9 @@ private static ColumnDateRange convertValidityDate(ConversionContext context, Va } private static SqlFilters collectConditionFilters( - List> conceptElements, CQTable cqTable, SqlFunctionProvider functionProvider) { + List> conceptElements, + CQTable cqTable, + SqlFunctionProvider functionProvider) { List conditions = new ArrayList<>(); conditions.addAll(collectConditions(conceptElements, cqTable, functionProvider)); @@ -175,15 +201,20 @@ private static SqlFilters collectConditionFilters( validityDateFilter = functionProvider.isNotEmptyValidityDate(validityDate); } - return new SqlFilters(ConnectorSqlSelects.none(), - WhereClauses.builder() - .preprocessingConditions(conditions) - .preprocessingCondition(ConditionUtil.wrap(validityDateFilter)) - .build() + return new SqlFilters( + ConnectorSqlSelects.none(), + WhereClauses.builder() + .preprocessingConditions(conditions) + .preprocessingCondition( + ConditionUtil.wrap(validityDateFilter)) + .build() ); } - private static List collectConditions(List> conceptElements, CQTable cqTable, SqlFunctionProvider functionProvider) { + private static List collectConditions( + List> conceptElements, + CQTable cqTable, + SqlFunctionProvider functionProvider) { List conditions = new ArrayList<>(); @@ -191,9 +222,9 @@ private static List collectConditions(List> co for (ConceptElement conceptElement : conceptElements) { - collectConditions(cqTable, conceptElement, functionProvider) - .reduce(WhereCondition::and) - .ifPresent(conditions::add); + collectConditions(cqTable, conceptElement, functionProvider).reduce(WhereCondition::and) + .ifPresent( + conditions::add); } return conditions; @@ -202,24 +233,34 @@ private static List collectConditions(List> co /** * Collects all conditions of a given {@link ConceptElement} by resolving the condition of the given node and all of its parent nodes. */ - private static Stream collectConditions(CQTable cqTable, ConceptElement conceptElement, SqlFunctionProvider functionProvider) { + private static Stream collectConditions( + CQTable cqTable, + ConceptElement conceptElement, + SqlFunctionProvider functionProvider) { if (!(conceptElement instanceof ConceptTreeChild child)) { return Stream.empty(); } - WhereCondition childCondition = child.getCondition().convertToSqlCondition(CTConditionContext.forConnector( - cqTable.getConnector().resolve(), functionProvider - )); + WhereCondition childCondition = child.getCondition() + .convertToSqlCondition( + CTConditionContext.forConnector( + cqTable.getConnector().resolve(), + functionProvider + )); return Stream.concat( - collectConditions(cqTable, child.getParent(), functionProvider), - Stream.of(childCondition) + collectConditions(cqTable, child.getParent(), functionProvider), + Stream.of(childCondition) ); } - private static Optional convertConnectorCondition(CQTable cqTable, SqlFunctionProvider functionProvider) { + private static Optional convertConnectorCondition( + CQTable cqTable, + SqlFunctionProvider functionProvider) { final Connector connector = cqTable.getConnector().resolve(); return Optional.ofNullable(connector.getCondition()) - .map(condition -> condition.convertToSqlCondition(CTConditionContext.forConnector(connector, functionProvider))); + .map( + condition -> condition.convertToSqlCondition( + CTConditionContext.forConnector(connector, functionProvider))); } private static SqlFilters dateRestrictionFilter(ConversionContext context, ColumnDateRange validityDate) { @@ -233,22 +274,29 @@ private static SqlFilters dateRestrictionFilter(ConversionContext context, Colum } return new SqlFilters( - ConnectorSqlSelects.none(), - WhereClauses.builder() - .eventFilters(conditions) - .build() + ConnectorSqlSelects.none(), + WhereClauses.builder().eventFilters(conditions).build() ); } - private static ConnectorSqlSelects createConceptColumnConnectorSqlSelects(CQConcept cqConcept, SelectContext selectContext) { - - - return cqConcept.getSelects().stream() - .map(SelectId::resolve) - .filter(select -> select instanceof ConceptColumnSelect) - .findFirst() - .map(select -> selectContext.getDialectBundle().getSelectConverter(select).connectorSelect(select, selectContext)) - .orElse(ConnectorSqlSelects.none()); + private static ConnectorSqlSelects createConceptColumnConnectorSqlSelects( + CQConcept cqConcept, + SelectContext selectContext) { + + + return cqConcept.getSelects() + .stream() + .map(SelectId::resolve) + .filter( + select -> select instanceof ConceptColumnSelect) + .findFirst() + .map( + select -> selectContext.getDialectBundle() + .getSelectConverter(select) + .connectorSelect( + select, + selectContext)) + .orElse(ConnectorSqlSelects.none()); } @Override @@ -260,16 +308,26 @@ public Class getConversionClass() { public ConversionContext convert(CQConcept cqConcept, ConversionContext context) { TablePath tablePath = new TablePath(cqConcept, context); - List convertedCQTables = cqConcept.getTables().stream() - .flatMap(cqTable -> convertCqTable(tablePath, cqConcept, cqTable, context).stream()) - .toList(); - - QueryStep joinedStep = QueryStepJoiner.joinSteps(convertedCQTables, ConqueryJoinType.OUTER_JOIN, DateAggregationAction.MERGE, context); + List convertedCQTables = cqConcept.getTables() + .stream() + .flatMap( + cqTable -> convertCqTable(tablePath, cqConcept, cqTable, context).stream()) + .toList(); + + QueryStep joinedStep = QueryStepJoiner.joinSteps( + convertedCQTables, + ConqueryJoinType.OUTER_JOIN, + DateAggregationAction.MERGE, + context); QueryStep lastConceptStep = finishConceptConversion(joinedStep, cqConcept, tablePath, context); return context.withQueryStep(lastConceptStep); } - private Optional convertCqTable(TablePath tablePath, CQConcept cqConcept, CQTable cqTable, ConversionContext context) { + private Optional convertCqTable( + TablePath tablePath, + CQConcept cqConcept, + CQTable cqTable, + ConversionContext context) { CQTableContext tableContext = createTableContext(tablePath, cqConcept, cqTable, context); Optional lastQueryStep = Optional.empty(); for (ConnectorCte queryStep : connectorCTEs) { @@ -283,7 +341,11 @@ private Optional convertCqTable(TablePath tablePath, CQConcept cqConc return lastQueryStep; } - private CQTableContext createTableContext(TablePath tablePath, CQConcept cqConcept, CQTable cqTable, ConversionContext conversionContext) { + private CQTableContext createTableContext( + TablePath tablePath, + CQConcept cqConcept, + CQTable cqTable, + ConversionContext conversionContext) { SqlIdColumns ids = convertIds(cqConcept, cqTable, conversionContext); ConnectorSqlTables connectorTables = tablePath.getConnectorTables(cqTable); @@ -293,35 +355,53 @@ private CQTableContext createTableContext(TablePath tablePath, CQConcept cqConce SqlFunctionProvider functionProvider = conversionContext.getFunctionProvider(); List allSqlFiltersForTable = new ArrayList<>(); - cqTable.getFilters().stream() - .map(filterValue -> filterValue.convertToSqlFilter(ids, conversionContext, connectorTables)) - .forEach(allSqlFiltersForTable::add); - - List> conceptElements = cqConcept.getElements().stream().>map(ConceptElementId::resolve).toList(); + cqTable.getFilters() + .stream() + .map( + filterValue -> filterValue.convertToSqlFilter(ids, conversionContext, connectorTables)) + .forEach( + allSqlFiltersForTable::add); + + List> conceptElements = cqConcept.getElements() + .stream() + .>map( + ConceptElementId::resolve) + .toList(); allSqlFiltersForTable.add(collectConditionFilters(conceptElements, cqTable, functionProvider)); allSqlFiltersForTable.add(dateRestrictionFilter(conversionContext, validityDateCalculation)); // convert selects - SelectContext selectContext = SelectContext.create(ids, Optional.of(validityDateCalculation.asValidityDateRange(connectorTables.getName())), connectorTables, conversionContext); + SelectContext selectContext = SelectContext.create( + ids, + Optional.of(validityDateCalculation.asValidityDateRange(connectorTables.getName())), + connectorTables, + conversionContext); List allSelectsForTable = new ArrayList<>(); ConnectorSqlSelects conceptColumnSelect = createConceptColumnConnectorSqlSelects(cqConcept, selectContext); allSelectsForTable.add(conceptColumnSelect); cqTable.getSelects() - .stream() - .map(SelectId::resolve) - .map(select -> selectContext.getDialectBundle().getSelectConverter(select).connectorSelect(select, selectContext)) - .forEach(allSelectsForTable::add); + .stream() + .map(SelectId::resolve) + .map( + select -> selectContext.getDialectBundle() + .getSelectConverter(select) + .connectorSelect( + select, + selectContext)) + .forEach(allSelectsForTable::add); return CQTableContext.builder() - .ids(ids) - .connector(connectorTables.getName()) - .rawValidityDate(validityDateCalculation) - .sqlSelects(allSelectsForTable) - .sqlFilters(allSqlFiltersForTable) - .connectorTables(connectorTables) - .conversionContext(conversionContext) - .build(); + .ids(ids) + .connector(connectorTables.getName()) + .rawValidityDate( + validityDateCalculation) + .sqlSelects(allSelectsForTable) + .sqlFilters(allSqlFiltersForTable) + .connectorTables( + connectorTables) + .conversionContext(conversionContext) + .build(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CTConditionContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CTConditionContext.java index 4f2badcce4a..564020e4cbf 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CTConditionContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CTConditionContext.java @@ -21,9 +21,11 @@ public static CTConditionContext forJoinTables(SqlFunctionProvider functionProvi public static CTConditionContext forConnector(Connector connector, SqlFunctionProvider functionProvider) { return new CTConditionContext( - connector.getColumn() != null ? name(connector.resolveTableId().getTable(), connector.getColumn().getColumn()) : null, - functionProvider, - true + connector.getColumn() != null ? name( + connector.resolveTableId().getTable(), + connector.getColumn().getColumn()) : null, + functionProvider, + true ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptCteStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptCteStep.java index 191dadec6fd..fb581b5623a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptCteStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptCteStep.java @@ -12,23 +12,21 @@ public enum ConceptCteStep implements CteStep { // connector - PREPROCESSING("preprocessing", null), - AGGREGATION_SELECT("group_select", PREPROCESSING), - JOIN_BRANCHES("join_branches", AGGREGATION_SELECT), - AGGREGATION_FILTER("group_filter", JOIN_BRANCHES), + PREPROCESSING("preprocessing", null), AGGREGATION_SELECT("group_select", PREPROCESSING), JOIN_BRANCHES( + "join_branches", + AGGREGATION_SELECT), AGGREGATION_FILTER("group_filter", JOIN_BRANCHES), // interval packing selects - UNNEST_DATE("unnested", null), - INTERVAL_PACKING_SELECTS("interval_packing_selects", null), + UNNEST_DATE("unnested", null), INTERVAL_PACKING_SELECTS("interval_packing_selects", null), // universal selects / final step UNIVERSAL_SELECTS("universal_selects", null); public static final Set MANDATORY_STEPS = Set.of( - PREPROCESSING, - AGGREGATION_SELECT, - JOIN_BRANCHES, - AGGREGATION_FILTER + PREPROCESSING, + AGGREGATION_SELECT, + JOIN_BRANCHES, + AGGREGATION_FILTER ); private final String suffix; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptSqlTables.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptSqlTables.java index 2fda1a3295c..f36a5be832d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptSqlTables.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptSqlTables.java @@ -13,10 +13,10 @@ public class ConceptSqlTables extends SqlTables { private final List connectorTables; public ConceptSqlTables( - String rootTable, - Map cteNameMap, - Map predecessorMap, - List connectorTables + String rootTable, + Map cteNameMap, + Map predecessorMap, + List connectorTables ) { super(rootTable, cteNameMap, predecessorMap); this.connectorTables = connectorTables; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCte.java index 0840b875e3d..e1d3dacb149 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCte.java @@ -23,10 +23,13 @@ protected Optional convert(CQTableContext tableContext, Optional cteNameMap, - Map predecessorMap, - boolean containsIntervalPacking, - boolean excludedFromTimeAggregation + Connector connector, + String connectorName, + String rootTable, + Map cteNameMap, + Map predecessorMap, + boolean containsIntervalPacking, + boolean excludedFromTimeAggregation ) { super(rootTable, cteNameMap, predecessorMap); this.connector = connector; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java index 6aa498302a8..206d93e5a26 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java @@ -31,7 +31,11 @@ public class FilterContext implements Context { @Nullable ConnectorSqlTables tables; - public static FilterContext forConceptConversion(SqlIdColumns ids, V value, ConversionContext conversionContext, ConnectorSqlTables tables) { + public static FilterContext forConceptConversion( + SqlIdColumns ids, + V value, + ConversionContext conversionContext, + ConnectorSqlTables tables) { return new FilterContext<>(ids, value, conversionContext, tables); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java index 45e03332c01..c004dbdf50f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java @@ -19,11 +19,11 @@ public class IntervalPackingSelectsCte { public static QueryStep forSelect( - QueryStep withAggregatedDaterange, - ColumnDateRange daterange, - SqlSelect select, - SqlTables tables, - DialectBundle dialectBundle + QueryStep withAggregatedDaterange, + ColumnDateRange daterange, + SqlSelect select, + SqlTables tables, + DialectBundle dialectBundle ) { List predecessors = List.of(withAggregatedDaterange); QueryStep directPredecessor = withAggregatedDaterange; @@ -31,45 +31,48 @@ public static QueryStep forSelect( // we need an additional predecessor to unnest the validity date if it is a single column range if (dialectBundle.supportsSingleColumnRanges()) { String unnestCteName = tables.cteName(ConceptCteStep.UNNEST_DATE); - directPredecessor = dialectBundle.getFunctionProvider().unnestDaterange(daterange, withAggregatedDaterange, unnestCteName); + directPredecessor = dialectBundle.getFunctionProvider() + .unnestDaterange( + daterange, + withAggregatedDaterange, + unnestCteName); predecessors = List.of(withAggregatedDaterange, directPredecessor); } Selects predecessorSelects = directPredecessor.getQualifiedSelects(); - Selects selects = Selects.builder() - .ids(predecessorSelects.getIds()) - .sqlSelect(select) - .build(); + Selects selects = Selects.builder().ids(predecessorSelects.getIds()).sqlSelect(select).build(); return QueryStep.builder() - .cteName(tables.cteName(ConceptCteStep.INTERVAL_PACKING_SELECTS)) - .selects(selects) - .fromTable(QueryStep.toTableLike(directPredecessor.getCteName())) - .groupBy(predecessorSelects.getIds().toFields()) - .predecessors(predecessors) - .build(); + .cteName(tables.cteName(ConceptCteStep.INTERVAL_PACKING_SELECTS)) + .selects( + selects) + .fromTable(QueryStep.toTableLike(directPredecessor.getCteName())) + .groupBy( + predecessorSelects.getIds().toFields()) + .predecessors(predecessors) + .build(); } public static QueryStep forConnector(QueryStep predecessor, CQTableContext cqTableContext) { return create( - predecessor, - cqTableContext.getSqlSelects().stream().flatMap(selects -> selects.getEventDateSelects().stream()).toList(), - cqTableContext.getConnectorTables(), - cqTableContext.getFunctionProvider() + predecessor, + cqTableContext.getSqlSelects().stream().flatMap(selects -> selects.getEventDateSelects().stream()).toList(), + cqTableContext.getConnectorTables(), + cqTableContext.getFunctionProvider() ); } public static QueryStep forConcept( - QueryStep predecessor, - SqlTables tables, - List sqlSelects, - ConversionContext conversionContext + QueryStep predecessor, + SqlTables tables, + List sqlSelects, + ConversionContext conversionContext ) { return create( - predecessor, - sqlSelects.stream().flatMap(selects -> selects.getEventDateSelects().stream()).toList(), - tables, - conversionContext.getFunctionProvider() + predecessor, + sqlSelects.stream().flatMap(selects -> selects.getEventDateSelects().stream()).toList(), + tables, + conversionContext.getFunctionProvider() ); } @@ -80,33 +83,34 @@ public static QueryStep forConcept( * Returns the given predecessor as is if the given list of interval packing selects is empty. */ private static QueryStep create( - QueryStep predecessor, - List intervalPackingSelects, - SqlTables tables, - SqlFunctionProvider functionProvider + QueryStep predecessor, + List intervalPackingSelects, + SqlTables tables, + SqlFunctionProvider functionProvider ) { if (intervalPackingSelects.isEmpty()) { return predecessor; } Optional validityDate = predecessor.getQualifiedSelects().getValidityDate(); - Preconditions.checkArgument(validityDate.isPresent(), "Can't create a IntervalPackingSelectsCte without a validity date present."); + Preconditions.checkArgument( + validityDate.isPresent(), + "Can't create a IntervalPackingSelectsCte without a validity date present."); // we need an additional predecessor to unnest the validity date if it is a single column range - Selects predecessorSelects = predecessor.getQualifiedSelects(); - Selects selects = Selects.builder() - .ids(predecessorSelects.getIds()) - .sqlSelects(intervalPackingSelects) - .build(); + Selects predecessorSelects = predecessor.getQualifiedSelects(); + Selects selects = Selects.builder().ids(predecessorSelects.getIds()).sqlSelects(intervalPackingSelects).build(); return QueryStep.builder() - .cteName(tables.cteName(ConceptCteStep.INTERVAL_PACKING_SELECTS)) - .selects(selects) - .fromTable(QueryStep.toTableLike(predecessor.getCteName())) - .groupBy(predecessorSelects.getIds().toFields()) - .predecessors(List.of()) - .build(); + .cteName(tables.cteName(ConceptCteStep.INTERVAL_PACKING_SELECTS)) + .selects( + selects) + .fromTable(QueryStep.toTableLike(predecessor.getCteName())) + .groupBy( + predecessorSelects.getIds().toFields()) + .predecessors(List.of()) + .build(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java index b2f6718dfd0..bb9edd2e27b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java @@ -60,15 +60,16 @@ protected QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { Optional validityDate; if (!tableContext.getConnectorTables().isWithIntervalPacking()) { validityDate = Optional.empty(); - } - else { + } else { IntervalPackingContext intervalPackingContext = createIntervalPackingContext(tableContext); IntervalPacker intervalPacker = tableContext.getConversionContext().getDialectBundle().getIntervalPacker(); QueryStep lastIntervalPackingStep = intervalPacker.aggregateAsValidityDate(intervalPackingContext); queriesToJoin.add(lastIntervalPackingStep); validityDate = lastIntervalPackingStep.getQualifiedSelects().getValidityDate(); - QueryStep intervalPackingSelectsStep = IntervalPackingSelectsCte.forConnector(lastIntervalPackingStep, tableContext); + QueryStep intervalPackingSelectsStep = IntervalPackingSelectsCte.forConnector( + lastIntervalPackingStep, + tableContext); if (intervalPackingSelectsStep != lastIntervalPackingStep) { queriesToJoin.add(intervalPackingSelectsStep); } @@ -80,40 +81,49 @@ protected QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { } // additional preceding tables - tableContext.allSqlSelects().stream() - .flatMap(sqlSelects -> sqlSelects.getAdditionalPredecessor().stream()) - .forEach(queriesToJoin::add); + tableContext.allSqlSelects() + .stream() + .flatMap( + sqlSelects -> sqlSelects.getAdditionalPredecessor().stream()) + .forEach(queriesToJoin::add); Selects selects = collectSelects(validityDate, queriesToJoin, tableContext); - TableLike fromTable = QueryStepJoiner.constructJoinedTable(queriesToJoin, ConqueryJoinType.OUTER_JOIN, tableContext.getConversionContext()); + TableLike fromTable = QueryStepJoiner.constructJoinedTable( + queriesToJoin, + ConqueryJoinType.OUTER_JOIN, + tableContext.getConversionContext()); - return QueryStep.builder() - .selects(selects) - .fromTable(fromTable) - .predecessors(queriesToJoin); + return QueryStep.builder().selects(selects).fromTable(fromTable).predecessors(queriesToJoin); } private static IntervalPackingContext createIntervalPackingContext(CQTableContext tableContext) { Selects predcessorSelects = tableContext.getPrevious().getQualifiedSelects(); return IntervalPackingContext.builder() - .ids(predcessorSelects.getIds()) - .daterange(tableContext.getValidityDate()) - .tables(tableContext.getConnectorTables()) - .build(); + .ids(predcessorSelects.getIds()) + .daterange( + tableContext.getValidityDate()) + .tables(tableContext.getConnectorTables()) + .build(); } - private static Selects collectSelects(Optional validityDate, List queriesToJoin, CQTableContext tableContext) { + private static Selects collectSelects( + Optional validityDate, + List queriesToJoin, + CQTableContext tableContext) { SqlIdColumns ids = QueryStepJoiner.coalesceIds(queriesToJoin); List mergedSqlSelects = QueryStepJoiner.mergeSelects(queriesToJoin); - Optional stratificationDate = tableContext.getPrevious().getQualifiedSelects().getStratificationDate(); + Optional stratificationDate = tableContext.getPrevious() + .getQualifiedSelects() + .getStratificationDate(); return Selects.builder() - .ids(ids) - .stratificationDate(stratificationDate) - .validityDate(validityDate) - .sqlSelects(mergedSqlSelects) - .build(); + .ids(ids) + .stratificationDate(stratificationDate) + .validityDate(validityDate) + .sqlSelects( + mergedSqlSelects) + .build(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java index 7556a1b8694..7823a324703 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java @@ -28,15 +28,18 @@ public ConceptCteStep cteStep() { @Override public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { - List forPreprocessing = tableContext.allSqlSelects().stream() - .flatMap(sqlSelects -> sqlSelects.getPreprocessingSelects().stream()) - .toList(); + List forPreprocessing = tableContext.allSqlSelects() + .stream() + .flatMap( + sqlSelects -> sqlSelects.getPreprocessingSelects().stream()) + .toList(); Selects preprocessingSelects = Selects.builder() - .ids(tableContext.getIds()) - .validityDate(Optional.of(tableContext.getValidityDate())) - .sqlSelects(forPreprocessing) - .build(); + .ids(tableContext.getIds()) + .validityDate( + Optional.of(tableContext.getValidityDate())) + .sqlSelects(forPreprocessing) + .build(); // All event-level where clauses are applied directly to the connector table while preprocessing. List conditions = new ArrayList<>(); @@ -51,61 +54,66 @@ public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { // The aliased secondary ID is selected in this CTE, so its root-table expression must be used in the WHERE clause. tableContext.getIds() - .getPredecessor() - .flatMap(SqlIdColumns::getSecondaryId) - .ifPresent(secondaryId -> conditions.add(secondaryId.isNotNull())); + .getPredecessor() + .flatMap(SqlIdColumns::getSecondaryId) + .ifPresent( + secondaryId -> conditions.add(secondaryId.isNotNull())); - QueryStep.QueryStepBuilder builder = QueryStep.builder() - .selects(preprocessingSelects) - .conditions(conditions); + QueryStep.QueryStepBuilder builder = QueryStep.builder().selects(preprocessingSelects).conditions(conditions); if (tableContext.getConversionContext().isWithStratification()) { return joinWithStratificationTable(forPreprocessing, conditions, tableContext); } - TableLike rootTable = QueryStep.toTableLike(tableContext.getConnectorTables().getPredecessor(ConceptCteStep.PREPROCESSING)); + TableLike rootTable = QueryStep.toTableLike( + tableContext.getConnectorTables().getPredecessor(ConceptCteStep.PREPROCESSING)); return builder.fromTable(rootTable); } private static QueryStep.QueryStepBuilder joinWithStratificationTable( - List preprocessingSelects, - List conditions, - CQTableContext tableContext + List preprocessingSelects, + List conditions, + CQTableContext tableContext ) { QueryStep stratificationTableCte = tableContext.getConversionContext().getStratificationTable(); Table stratificationTable = DSL.table(DSL.name(stratificationTableCte.getCteName())); Selects stratificationSelects = stratificationTableCte.getQualifiedSelects(); SqlIdColumns stratificationIds = stratificationSelects.getIds(); - SqlIdColumns rootTableIds = tableContext.getIds().getPredecessor().orElseThrow(() -> new IllegalStateException( - "Id's should have been qualified during conversion and thus have a predecessor") - ); + SqlIdColumns rootTableIds = tableContext.getIds() + .getPredecessor() + .orElseThrow( + () -> new IllegalStateException( + "Id's should have been qualified during conversion and thus have a predecessor") + ); List idConditions = stratificationIds.join(rootTableIds); // join full stratification with connector table on all ID's from prerequisite query SqlFunctionProvider functionProvider = tableContext.getConversionContext().getFunctionProvider(); - ColumnDateRange stratificationDate = stratificationSelects.getStratificationDate().orElseThrow(() -> new IllegalStateException( - "Stratification table must provide a stratification date" - )); + ColumnDateRange stratificationDate = stratificationSelects.getStratificationDate() + .orElseThrow( + () -> new IllegalStateException( + "Stratification table must provide a stratification date" + )); // Both expressions are available from the joined source tables; do not reference aliases produced by this SELECT. conditions.add(functionProvider.dateRestriction(stratificationDate, tableContext.getRawValidityDate())); - Table connectorTable = DSL.table(DSL.name(tableContext.getConnectorTables().getPredecessor(ConceptCteStep.PREPROCESSING))); + Table connectorTable = DSL.table( + DSL.name(tableContext.getConnectorTables().getPredecessor(ConceptCteStep.PREPROCESSING))); TableLike joinedTable = functionProvider.innerJoin(connectorTable, stratificationTable, idConditions); Selects selects = Selects.builder() - .ids(stratificationSelects.getIds()) - .validityDate(Optional.of(tableContext.getValidityDate())) - .stratificationDate(stratificationSelects.getStratificationDate()) - .sqlSelects(preprocessingSelects) - .build(); - - return QueryStep.builder() - .selects(selects) - .fromTable(joinedTable) - .conditions(conditions); + .ids(stratificationSelects.getIds()) + .validityDate( + Optional.of(tableContext.getValidityDate())) + .stratificationDate( + stratificationSelects.getStratificationDate()) + .sqlSelects(preprocessingSelects) + .build(); + + return QueryStep.builder().selects(selects).fromTable(joinedTable).conditions(conditions); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java index 0ccbe641510..3aba62e6d59 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java @@ -37,53 +37,77 @@ class TablePath { public TablePath(CQConcept cqConcept, ConversionContext context) { this.cqConcept = cqConcept; this.context = context; - cqConcept.getTables().forEach(cqTable -> this.connectorTableMap.put(cqTable, createConnectorTables(cqConcept, cqTable, context))); + cqConcept.getTables() + .forEach( + cqTable -> this.connectorTableMap.put(cqTable, createConnectorTables(cqConcept, cqTable, context))); } public ConnectorSqlTables getConnectorTables(CQTable cqTable) { return connectorTableMap.get(cqTable); } - private static ConnectorSqlTables createConnectorTables(CQConcept cqConcept, CQTable cqTable, ConversionContext context) { + private static ConnectorSqlTables createConnectorTables( + CQConcept cqConcept, + CQTable cqTable, + ConversionContext context) { - String connectorName = context.getNameGenerator().conceptConnectorName(cqConcept, cqTable.getConnector().resolve(), context.getSqlPrintSettings() - .getLocale()); + String connectorName = context.getNameGenerator() + .conceptConnectorName( + cqConcept, + cqTable.getConnector().resolve(), + context.getSqlPrintSettings().getLocale()); TablePathInfo tableInfo = collectConnectorTables(cqConcept, cqTable, context); - Map cteNameMap = CteStep.createCteNameMap(tableInfo.getMappings().keySet(), connectorName, context.getNameGenerator()); + Map cteNameMap = CteStep.createCteNameMap( + tableInfo.getMappings().keySet(), + connectorName, + context.getNameGenerator()); return new ConnectorSqlTables( - cqTable.getConnector().resolve(), - connectorName, - tableInfo.getRootTable(), - cteNameMap, - tableInfo.getMappings(), - tableInfo.isContainsIntervalPacking(), - tableInfo.isExcludedFromTimeAggregation() + cqTable.getConnector().resolve(), + connectorName, + tableInfo.getRootTable(), + cteNameMap, + tableInfo.getMappings(), + tableInfo.isContainsIntervalPacking(), + tableInfo.isExcludedFromTimeAggregation() ); } public ConceptSqlTables createConceptTables(QueryStep predecessor) { TablePathInfo tableInfo = collectConceptTables(predecessor); - String conceptName = context.getNameGenerator().conceptName(cqConcept, context.getSqlPrintSettings().getLocale()); - Map cteNameMap = CteStep.createCteNameMap(tableInfo.getMappings().keySet(), conceptName, context.getNameGenerator()); + String conceptName = context.getNameGenerator() + .conceptName( + cqConcept, + context.getSqlPrintSettings().getLocale()); + Map cteNameMap = CteStep.createCteNameMap( + tableInfo.getMappings().keySet(), + conceptName, + context.getNameGenerator()); List connectorSqlTables = this.connectorTableMap.values().stream().toList(); return new ConceptSqlTables( - tableInfo.getRootTable(), - cteNameMap, - tableInfo.getMappings(), - connectorSqlTables + tableInfo.getRootTable(), + cteNameMap, + tableInfo.getMappings(), + connectorSqlTables ); } - private static TablePathInfo collectConnectorTables(CQConcept cqConcept, CQTable cqTable, ConversionContext context) { + private static TablePathInfo collectConnectorTables( + CQConcept cqConcept, + CQTable cqTable, + ConversionContext context) { TablePathInfo tableInfo = new TablePathInfo(); tableInfo.setRootTable(cqTable.getConnector().resolve().resolveTableId().getTable()); tableInfo.addWithDefaultMapping(MANDATORY_STEPS); - boolean eventDateSelectsPresent = cqTable.getSelects().stream().map(SelectId::resolve).anyMatch(Select::isEventDateSelect); + boolean eventDateSelectsPresent = cqTable.getSelects() + .stream() + .map(SelectId::resolve) + .anyMatch( + Select::isEventDateSelect); // no validity date aggregation necessary if (!cqConcept.isAggregateEventDates() && !eventDateSelectsPresent) { return tableInfo; @@ -104,15 +128,19 @@ private static TablePathInfo collectConnectorTables(CQConcept cqConcept, CQTable // interval packing selects required with optional unnest step if (context.getDialectBundle().supportsSingleColumnRanges()) { - tableInfo.addMappings(Map.of( - UNNEST_DATE, INTERVAL_COMPLETE, - INTERVAL_PACKING_SELECTS, UNNEST_DATE - )); - } - else { - tableInfo.addMappings(Map.of( - INTERVAL_PACKING_SELECTS, INTERVAL_COMPLETE - )); + tableInfo.addMappings( + Map.of( + UNNEST_DATE, + INTERVAL_COMPLETE, + INTERVAL_PACKING_SELECTS, + UNNEST_DATE + )); + } else { + tableInfo.addMappings( + Map.of( + INTERVAL_PACKING_SELECTS, + INTERVAL_COMPLETE + )); } return tableInfo; @@ -130,16 +158,15 @@ private TablePathInfo collectConceptTables(QueryStep predecessor) { } Preconditions.checkArgument( - predecessor.getSelects().getValidityDate().isPresent(), - "Can not convert Selects that require interval packing without a validity date present after converting (a) connector(s)" + predecessor.getSelects().getValidityDate().isPresent(), + "Can not convert Selects that require interval packing without a validity date present after converting (a) connector(s)" ); // universal event date selects required with optional additional unnest step if (context.getDialectBundle().supportsSingleColumnRanges()) { tableInfo.addRootTableMapping(UNNEST_DATE); tableInfo.addMappings(Map.of(INTERVAL_PACKING_SELECTS, UNNEST_DATE)); - } - else { + } else { tableInfo.addRootTableMapping(INTERVAL_PACKING_SELECTS); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AggregationMode.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AggregationMode.java index 45e6f3b03f4..1d02b61e4f3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AggregationMode.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AggregationMode.java @@ -7,6 +7,5 @@ * Determines if the date that is getting interval-packed is stored as validity date of {@link Selects} or as arbitrary {@link SqlSelect}. */ enum AggregationMode { - VALIDITY_DATE, - ARBITRARY_SELECT + VALIDITY_DATE, ARBITRARY_SELECT } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java index 178136ea1a1..29ad2afb05d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java @@ -45,56 +45,78 @@ private QueryStep createPreviousEndStep(IntervalPackingContext context, Aggregat ColumnDateRange daterange = context.getDaterange().qualify(sourceTableName); Field previousEnd = DSL.max(daterange.getEnd()) - .over(DSL.partitionBy(ids.toFields()) - .orderBy(daterange.getStart(), daterange.getEnd()) - .rowsBetweenUnboundedPreceding() - .andPreceding(1)) - .as(IntervalPacker.PREVIOUS_END_FIELD_NAME); - - List qualifiedSelects = new ArrayList<>(QualifyingUtil.qualify(context.getCarryThroughSelects(), sourceTableName)); - qualifiedSelects.add(new FieldWrapper<>(previousEnd, daterange.getStart().getName(), daterange.getEnd().getName())); + .over( + DSL.partitionBy(ids.toFields()) + .orderBy( + daterange.getStart(), + daterange.getEnd()) + .rowsBetweenUnboundedPreceding() + .andPreceding(1)) + .as( + IntervalPacker.PREVIOUS_END_FIELD_NAME); + + List qualifiedSelects = new ArrayList<>( + QualifyingUtil.qualify(context.getCarryThroughSelects(), sourceTableName)); + qualifiedSelects.add( + new FieldWrapper<>(previousEnd, daterange.getStart().getName(), daterange.getEnd().getName())); Selects previousEndSelects = buildSelects(ids, daterange, qualifiedSelects, aggregationMode); return QueryStep.builder() - .cteName(context.getTables().cteName(IntervalPackingCteStep.PREVIOUS_END)) - .selects(previousEndSelects) - .fromTable(QueryStep.toTableLike(sourceTableName)) - .predecessors(Optional.ofNullable(context.getPredecessor()).stream().toList()) - .build(); + .cteName(context.getTables().cteName(IntervalPackingCteStep.PREVIOUS_END)) + .selects( + previousEndSelects) + .fromTable(QueryStep.toTableLike(sourceTableName)) + .predecessors( + Optional.ofNullable(context.getPredecessor()).stream().toList()) + .build(); } - private QueryStep createRangeIndexStep(QueryStep previousEndStep, IntervalPackingContext context, AggregationMode aggregationMode) { + private QueryStep createRangeIndexStep( + QueryStep previousEndStep, + IntervalPackingContext context, + AggregationMode aggregationMode) { String previousEndCteName = previousEndStep.getCteName(); Selects previousEndSelects = previousEndStep.getQualifiedSelects(); SqlIdColumns ids = previousEndSelects.getIds(); ColumnDateRange daterange = context.getDaterange().qualify(previousEndCteName); - Field previousEnd = DSL.field(DSL.name(previousEndCteName, IntervalPacker.PREVIOUS_END_FIELD_NAME), Date.class); - - Field rangeIndex = - DSL.sum( - DSL.when(daterange.getStart().greaterThan(previousEnd), DSL.inline(1)) - .otherwise(DSL.inline(null, Integer.class))) - .over(DSL.partitionBy(ids.toFields()) - .orderBy(daterange.getStart(), daterange.getEnd()) - .rowsUnboundedPreceding()) - .as(IntervalPacker.RANGE_INDEX_FIELD_NAME); - - List qualifiedSelects = new ArrayList<>(QualifyingUtil.qualify(context.getCarryThroughSelects(), previousEndCteName)); + Field previousEnd = DSL.field( + DSL.name(previousEndCteName, IntervalPacker.PREVIOUS_END_FIELD_NAME), + Date.class); + + Field rangeIndex = DSL.sum( + DSL.when(daterange.getStart().greaterThan(previousEnd), DSL.inline(1)) + .otherwise( + DSL.inline(null, Integer.class))) + .over( + DSL.partitionBy(ids.toFields()) + .orderBy( + daterange.getStart(), + daterange.getEnd()) + .rowsUnboundedPreceding()) + .as(IntervalPacker.RANGE_INDEX_FIELD_NAME); + + List qualifiedSelects = new ArrayList<>( + QualifyingUtil.qualify(context.getCarryThroughSelects(), previousEndCteName)); qualifiedSelects.add(new FieldWrapper<>(rangeIndex)); Selects rangeIndexSelects = buildSelects(ids, daterange, qualifiedSelects, aggregationMode); return QueryStep.builder() - .cteName(context.getTables().cteName(IntervalPackingCteStep.RANGE_INDEX)) - .selects(rangeIndexSelects) - .fromTable(QueryStep.toTableLike(previousEndCteName)) - .predecessors(List.of(previousEndStep)) - .build(); + .cteName(context.getTables().cteName(IntervalPackingCteStep.RANGE_INDEX)) + .selects( + rangeIndexSelects) + .fromTable(QueryStep.toTableLike(previousEndCteName)) + .predecessors( + List.of(previousEndStep)) + .build(); } - private QueryStep createIntervalCompleteStep(QueryStep rangeIndexStep, IntervalPackingContext context, AggregationMode aggregationMode) { + private QueryStep createIntervalCompleteStep( + QueryStep rangeIndexStep, + IntervalPackingContext context, + AggregationMode aggregationMode) { String rangeIndexCteName = rangeIndexStep.getCteName(); Selects rangeIndexSelects = rangeIndexStep.getQualifiedSelects(); @@ -104,7 +126,9 @@ private QueryStep createIntervalCompleteStep(QueryStep rangeIndexStep, IntervalP Field rangeStart = DSL.min(daterange.getStart()).as(daterange.getStart().getName()); Field rangeEnd = DSL.max(daterange.getEnd()).as(daterange.getEnd().getName()); ColumnDateRange minMax = ColumnDateRange.of(rangeStart, rangeEnd); - Field rangeIndex = DSL.field(DSL.name(rangeIndexCteName, IntervalPacker.RANGE_INDEX_FIELD_NAME), BigDecimal.class); + Field rangeIndex = DSL.field( + DSL.name(rangeIndexCteName, IntervalPacker.RANGE_INDEX_FIELD_NAME), + BigDecimal.class); List qualifiedSelects = QualifyingUtil.qualify(context.getCarryThroughSelects(), rangeIndexCteName); Selects intervalCompleteSelects = buildSelects(ids, minMax, qualifiedSelects, aggregationMode); @@ -116,22 +140,24 @@ private QueryStep createIntervalCompleteStep(QueryStep rangeIndexStep, IntervalP qualifiedSelects.stream().flatMap(sqlSelect -> sqlSelect.toFields().stream()).forEach(groupBySelects::add); return QueryStep.builder() - .cteName(context.getTables().cteName(IntervalPackingCteStep.INTERVAL_COMPLETE)) - .selects(intervalCompleteSelects) - .fromTable(QueryStep.toTableLike(rangeIndexCteName)) - .predecessors(List.of(rangeIndexStep)) - .groupBy(groupBySelects) - .build(); + .cteName( + context.getTables().cteName(IntervalPackingCteStep.INTERVAL_COMPLETE)) + .selects( + intervalCompleteSelects) + .fromTable(QueryStep.toTableLike(rangeIndexCteName)) + .predecessors( + List.of(rangeIndexStep)) + .groupBy(groupBySelects) + .build(); } private static Selects buildSelects( - SqlIdColumns ids, - ColumnDateRange daterange, - List carryThroughSelects, - AggregationMode aggregationMode + SqlIdColumns ids, + ColumnDateRange daterange, + List carryThroughSelects, + AggregationMode aggregationMode ) { - Selects.SelectsBuilder builder = Selects.builder() - .ids(ids); + Selects.SelectsBuilder builder = Selects.builder().ids(ids); switch (aggregationMode) { case VALIDITY_DATE -> builder.validityDate(Optional.of(daterange)); case ARBITRARY_SELECT -> builder.sqlSelect(daterange); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java index d644a635e94..fd6d0f70c4f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java @@ -2,7 +2,6 @@ import java.util.Collections; import java.util.List; - import javax.annotation.Nullable; import com.bakdata.conquery.sql.conversion.Context; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java index ef5d068653c..15bebbb791d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java @@ -16,9 +16,9 @@ @AllArgsConstructor public enum IntervalPackingCteStep implements CteStep { - PREVIOUS_END("previous_end", null), - RANGE_INDEX("range_index", PREVIOUS_END), - INTERVAL_COMPLETE("interval_complete", RANGE_INDEX); + PREVIOUS_END("previous_end", null), RANGE_INDEX("range_index", PREVIOUS_END), INTERVAL_COMPLETE( + "interval_complete", + RANGE_INDEX); private static final Set STEPS = Set.of(values()); @@ -31,11 +31,13 @@ public enum IntervalPackingCteStep implements CteStep { public static SqlTables createTables(QueryStep predecessor, Context context) { String rootTable = predecessor.getCteName(); - Set requiredSteps = context.getDialectBundle().supportsSingleColumnRanges() - ? Set.of(INTERVAL_COMPLETE) - : Set.of(values()); + Set requiredSteps = context.getDialectBundle().supportsSingleColumnRanges() ? Set.of( + INTERVAL_COMPLETE) : Set.of(values()); - Map cteNameMap = CteStep.createCteNameMap(requiredSteps, rootTable, context.getNameGenerator()); + Map cteNameMap = CteStep.createCteNameMap( + requiredSteps, + rootTable, + context.getNameGenerator()); Map predecessorMap = CteStep.getDefaultPredecessorMap(requiredSteps); return new SqlTables(rootTable, cteNameMap, predecessorMap); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java index d0596962c30..7b89485ff9f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java @@ -32,7 +32,6 @@ import com.bakdata.conquery.sql.conversion.query.SecondaryIdQueryConverter; import com.bakdata.conquery.sql.conversion.query.TableExportQueryConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; -import com.bakdata.conquery.sql.execution.SqlCDateSetParser; import org.jooq.DSLContext; import org.jooq.Field; import org.jooq.SQLDialect; @@ -42,16 +41,18 @@ public interface DialectBundle { private static > List customize(List defaults, List substitutes) { Map, C> substituteMap = getSubstituteMap(substitutes); return defaults.stream() - .map(converter -> substituteMap.getOrDefault(converter.getConversionClass(), converter)) - .toList(); + .map( + converter -> substituteMap.getOrDefault(converter.getConversionClass(), converter)) + .toList(); } private static > Map, C> getSubstituteMap(List substitutes) { return substitutes.stream() - .collect(Collectors.toMap( - Converter::getConversionClass, - Function.identity() - )); + .collect( + Collectors.toMap( + Converter::getConversionClass, + Function.identity() + )); } ResultSetProcessor getResultSetProcessor(ConqueryConfig config); @@ -87,31 +88,32 @@ default List> getDefaultNodeConverters(DSLCon FormConversionHelper formConversionUtil = new FormConversionHelper(queryStepTransformer); return List.of( - new CQDateRestrictionConverter(), - new CQAndConverter(), - new CQOrConverter(), - new CQNegationConverter(), - new CQYesConverter(), - new CQConceptConverter(), - new CQExternalConverter(), - new CQReusedQueryConverter(), - new ConceptQueryConverter(queryStepTransformer), - new SecondaryIdQueryConverter(), - new AbsoluteFormQueryConverter(formConversionUtil), - new EntityDateQueryConverter(formConversionUtil), - new RelativFormQueryConverter(formConversionUtil), - new TableExportQueryConverter(queryStepTransformer) + new CQDateRestrictionConverter(), + new CQAndConverter(), + new CQOrConverter(), + new CQNegationConverter(), + new CQYesConverter(), + new CQConceptConverter(), + new CQExternalConverter(), + new CQReusedQueryConverter(), + new ConceptQueryConverter(queryStepTransformer), + new SecondaryIdQueryConverter(), + new AbsoluteFormQueryConverter(formConversionUtil), + new EntityDateQueryConverter(formConversionUtil), + new RelativFormQueryConverter(formConversionUtil), + new TableExportQueryConverter(queryStepTransformer) ); } - default Map, ? extends SelectConverter> getSelectConverterOverrides(){ + default Map, ? extends SelectConverter> getSelectConverterOverrides() { return Collections.emptyMap(); } - default SelectConverter maybeOverride = (SelectConverter getSelectConverter(Select select) { + SelectConverter) getSelectConverterOverrides().get( + select.getClass()); - if (maybeOverride != null){ + if (maybeOverride != null) { return maybeOverride; } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/Interval.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/Interval.java index 20eece3642a..fe274110159 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/Interval.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/Interval.java @@ -7,11 +7,8 @@ @RequiredArgsConstructor public enum Interval { - ONE_YEAR_INTERVAL(1), - YEAR_AS_DAYS_INTERVAL(365), - QUARTER_INTERVAL(3), - NINETY_DAYS_INTERVAL(90), - ONE_DAY_INTERVAL(1); + ONE_YEAR_INTERVAL(1), YEAR_AS_DAYS_INTERVAL(365), QUARTER_INTERVAL(3), NINETY_DAYS_INTERVAL(90), ONE_DAY_INTERVAL( + 1); public static final int DAYS_PER_YEAR = Interval.YEAR_AS_DAYS_INTERVAL.getAmount(); public static final int DAYS_PER_QUARTER = Interval.NINETY_DAYS_INTERVAL.getAmount(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java index 175ca894a52..3913ddc5e24 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDateAggregator.java @@ -17,14 +17,16 @@ public interface SqlDateAggregator { * @param carryThroughSelects The selects that should be carried through the date aggregation process. They remain unchanged. */ QueryStep apply( - QueryStep joinedStep, - List carryThroughSelects, - DateAggregationDates dateAggregationDates, - DateAggregationAction dateAggregationAction, - ConversionContext conversionContext + QueryStep joinedStep, + List carryThroughSelects, + DateAggregationDates dateAggregationDates, + DateAggregationAction dateAggregationAction, + ConversionContext conversionContext ); - ColumnDateRange getAggregatedValidityDate(DateAggregationDates dateAggregationDates, DateAggregationAction dateAggregationAction); + ColumnDateRange getAggregatedValidityDate( + DateAggregationDates dateAggregationDates, + DateAggregationAction dateAggregationAction); /** * Inverts the validity date of the given base step. diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java index 0a2b1552fdc..577c016fe57 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java @@ -43,16 +43,15 @@ public interface SqlFunctionProvider { * Create database specific representation of the input list, such that it can be read by the respective {@link ResultSetProcessor}. */ default Field asArrayRepr(List value) { - return field(value.stream() - .map(DSL::inline) - .map(Field::toString) - .collect(Collectors.joining(SQL_UNIT_SEPARATOR)), Object.class + return field( + value.stream().map(DSL::inline).map(Field::toString).collect(Collectors.joining(SQL_UNIT_SEPARATOR)), + Object.class ); } Collection> orderByValidityDates( - Function, ? extends SortField> ordering, - List> validityDateFields); + Function, ? extends SortField> ordering, + List> validityDateFields); /** * Return date-Field for the lowest representable date. This is specific per Database engine. @@ -101,16 +100,18 @@ default Table getNoOpTable() { * Creates a list of {@link ColumnDateRange}s for each {@link CDateRange} of the given {@link CDateSet}. Each {@link ColumnDateRange} will be aliased with * the same given {@link SharedAliases}. */ - default List forCDateSet(CDateSet dateset, SharedAliases alias){ + default List forCDateSet(CDateSet dateset, SharedAliases alias) { if (dateset.isEmpty()) { // Need to explicitly provide an empty result return List.of(emptyColumnDateRange().as(alias.getAlias())); } - return dateset.asRanges().stream() - .map(this::forCDateRange) - .map(dateRange -> dateRange.as(alias.getAlias())) - .toList(); + return dateset.asRanges() + .stream() + .map(this::forCDateRange) + .map( + dateRange -> dateRange.as(alias.getAlias())) + .toList(); } /** @@ -220,14 +221,17 @@ default Condition isNotEmptyValidityDate(ValidityDate validityDate) { */ Field yearQuarter(Field dateField); - default Field stringAggregation(Field stringField, Field delimiter, List> orderByFields) { + default Field stringAggregation( + Field stringField, + Field delimiter, + List> orderByFields) { return field( - "{0}({1}, {2} {3})", - String.class, - keyword("string_agg"), - stringField, - delimiter, - orderBy(orderByFields) + "{0}({1}, {2} {3})", + String.class, + keyword("string_agg"), + stringField, + delimiter, + orderBy(orderByFields) ); } @@ -237,12 +241,11 @@ default Field stringAggregation(Field stringField, Field * Render an array for Conquery processing. */ default Field arrayOut(List> fields) { - String concatenated = - fields.stream() - // if a field is null, the whole concatenation would be null - but we just want to skip this field in this case, - // thus concat an empty string - .map(Field::toString) - .collect(Collectors.joining(SQL_UNIT_SEPARATOR)); + String concatenated = fields.stream() + // if a field is null, the whole concatenation would be null - but we just want to skip this field in this case, + // thus concat an empty string + .map(Field::toString) + .collect(Collectors.joining(SQL_UNIT_SEPARATOR)); return field(concatenated, String.class); } @@ -268,15 +271,24 @@ default Condition in(Field column, String[] values) { return column.in(values); } - default TableOnConditionStep innerJoin(Table leftPart, Table rightPart, List joinConditions) { + default TableOnConditionStep innerJoin( + Table leftPart, + Table rightPart, + List joinConditions) { return leftPart.innerJoin(rightPart).on(joinConditions.toArray(Condition[]::new)); } - default TableOnConditionStep fullOuterJoin(Table leftPart, Table rightPart, List joinConditions) { + default TableOnConditionStep fullOuterJoin( + Table leftPart, + Table rightPart, + List joinConditions) { return leftPart.fullOuterJoin(rightPart).on(joinConditions.toArray(Condition[]::new)); } - default TableOnConditionStep leftJoin(Table leftPart, Table rightPart, List joinConditions) { + default TableOnConditionStep leftJoin( + Table leftPart, + Table rightPart, + List joinConditions) { return leftPart.leftJoin(rightPart).on(joinConditions.toArray(Condition[]::new)); } @@ -311,11 +323,11 @@ default Field externalId(String id) { * Any condition that is acceptable for the specific database on a join. * (e.g. Hana does not like `true`) */ - default Condition unconditionalJoinCondition(){ + default Condition unconditionalJoinCondition() { return noCondition(); } - default Field isNull(Field field){ + default Field isNull(Field field) { return field.isNull(); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java index 9efe1053025..497c0eb3f87 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDialectBundle.java @@ -19,7 +19,6 @@ import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.execution.ResultSetProcessor; -import com.bakdata.conquery.sql.execution.SqlCDateSetParser; import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; import org.jooq.Field; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java index fb6ddc89a4f..048b9773dd8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java @@ -16,29 +16,38 @@ public class ClickhouseDistinctSelectConverter implements SelectConverter selectContext) { + public ConnectorSqlSelects connectorSelect( + DistinctSelect distinctSelect, + SelectContext selectContext) { String alias = selectContext.getNameGenerator().selectName(distinctSelect); ConnectorSqlTables tables = selectContext.getTables(); - SingleColumnSqlSelect preprocessingSelect = - MappableSingleColumnSelect.getSubstringSelect(distinctSelect.getColumn().get(), distinctSelect.getSubstringRange(), selectContext, alias); + SingleColumnSqlSelect preprocessingSelect = MappableSingleColumnSelect.getSubstringSelect( + distinctSelect.getColumn().get(), + distinctSelect.getSubstringRange(), + selectContext, + alias); String preprocessingTable = selectContext.getTables().cteName(ConceptCteStep.PREPROCESSING); SingleColumnSqlSelect qualified = preprocessingSelect.qualify(preprocessingTable); - FieldWrapper grouped = - new FieldWrapper<>(field("arrayFilter(x -> x <> '' and x is not null, groupUniqArray({0}))", Object.class, qualified.select()).as(alias), - qualified.select().getName() - ); + FieldWrapper grouped = new FieldWrapper<>( + field( + "arrayFilter(x -> x <> '' and x is not null, groupUniqArray({0}))", + Object.class, + qualified.select()).as(alias), + qualified.select().getName() + ); SingleColumnSqlSelect finalSelect = grouped.qualify(tables.cteName(ConceptCteStep.AGGREGATION_SELECT)); return ConnectorSqlSelects.builder() - .preprocessingSelect(preprocessingSelect) - .aggregationSelect(grouped) - .finalSelect(finalSelect) - .build(); + .preprocessingSelect(preprocessingSelect) + .aggregationSelect( + grouped) + .finalSelect(finalSelect) + .build(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java index 0131145f011..3f97a1eda51 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseFunctionProvider.java @@ -26,316 +26,325 @@ public class ClickhouseFunctionProvider implements SqlFunctionProvider { - public static final Integer MIN_DATE_VALUE = -25567; - public static final Integer MAX_DATE_VALUE = 24855; - private static final String ANY_CHAR_REGEX = ".*"; - - @Override - public String getAnyCharRegex() { - return ANY_CHAR_REGEX; - } - - @Override - public Field asArrayRepr(List value) { - return array(value.toArray()); - } - - - @Override - public Field externalId(String id) { - // This is only used in CQExternal to define external Ids. - // Without explicitly tagging this value as Nullable, the Clickhouse QueryPlanner assumes that this it can never be null. - // But in the case of an anti-join (our Negation), this field will be sometimes null, while not testing as `is null` = true - return field("{0}::Nullable(String)", String.class, inline(id, String.class)); - } - - @Override - public Condition dateRestriction(ColumnDateRange dateRestriction, ColumnDateRange daterange) { - Condition dateRestrictionStartsBeforeDate = dateRestriction.getStart().lessThan(daterange.getEnd()); - Condition dateRestrictionEndsAfterDate = dateRestriction.getEnd().greaterThan(daterange.getStart()); - - return condition(dateRestrictionStartsBeforeDate.and(dateRestrictionEndsAfterDate)); - } - - - @Override - public ColumnDateRange forCDateRange(CDateRange daterange) { - - Field startDateExpression = getMinDateExpression(); - Field endDateExpression = getMaxDateExpression(); - - if (daterange.hasLowerBound()) { - startDateExpression = inline(Date.valueOf(daterange.getMin())); - } - if (daterange.hasUpperBound()) { - endDateExpression = inline(Date.valueOf(daterange.getMax().plusDays(1))); - } - - return ColumnDateRange.of(startDateExpression, endDateExpression); - } - - @Override - public Field toDateField(String dateExpression) { - return function( - "toDate", - Date.class, - inline(dateExpression), - inline(DEFAULT_DATE_FORMAT) - ); - } - - @Override - public Condition isNotEmptyDateRange(ColumnDateRange columnDateRange) { - Condition startNotMin = columnDateRange.getStart().notEqual(getMinDateExpression()); - Condition endNotMax = columnDateRange.getEnd().notEqual(getMaxDateExpression()); - return condition(startNotMin.and(endNotMax).neg()); - } + public static final Integer MIN_DATE_VALUE = -25567; + public static final Integer MAX_DATE_VALUE = 24855; + private static final String ANY_CHAR_REGEX = ".*"; @Override - public ColumnDateRange forValidityDate(ValidityDate validityDate) { - return toColumnDateRange(validityDate); - } - - - @Override - public ColumnDateRange allRange() { - return ColumnDateRange.of(getMinDateExpression(), getMaxDateExpression()); - } - - @Override - public Field anyValue(Field field) { - return DSL.anyValue(field); - } - - private ColumnDateRange toColumnDateRange(ValidityDate validityDate) { - - String tableName = validityDate.getConnector().resolveTableId().getTable(); - - Column startColumn; - Column endColumn; - - // if no end column is present, the only existing column is both start and end of the date range - if (validityDate.isSingleColumnDaterange()) { - Column column = validityDate.getColumn().resolve(); - startColumn = column; - endColumn = column; - } else { - startColumn = validityDate.getStartColumn().resolve(); - endColumn = validityDate.getEndColumn().resolve(); - } - - return ofStartAndEnd(tableName, startColumn, endColumn); - } - - - @Override - public Field arrayOut(List> fields) { - return field("arrayFilter(x -> x <> '', {0})", Object.class, array(fields)); - } - - private ColumnDateRange ofStartAndEnd(String tableName, Column startColumn, Column endColumn) { - - // Since coalesce makes Clickhouse certain, that the field is not nullable, it will do silly stuff with it down the line: - // missing values (for example in outer-joins) will be coerced to 0 = 01-01-1970, which is clearly not correct - // Therefore we tag the values as Nullable again to make Clickhouse show some respect - - Field rangeStart = field("{0}::Nullable(Date32)", Date.class, coalesce( - field(name(tableName, startColumn.getName()), Date.class), - getMinDateExpression() - ) - ); - // when aggregating date ranges, we want to treat the last day of the range as excluded, - // so when using the date value of the end column, we add +1 day as end of the date range - Field rangeEnd = field("{0}::Nullable(Date32)", Date.class, coalesce( - addDays(field(name(tableName, endColumn.getName()), Date.class), inline(1)), - getMaxDateExpression() - ) - ); - - return ColumnDateRange.of(rangeStart, rangeEnd); - } - - @Override - public Field addDays(Field dateColumn, Field amountOfDays) { - return function( - "addDays", - Date.class, - dateColumn, - amountOfDays - ); - } - - @Override - public ColumnDateRange forValidityDate(ValidityDate validityDate, CDateRange dateRestriction) { - - ColumnDateRange validityDateRange = toColumnDateRange(validityDate); - ColumnDateRange restriction = toColumnDateRange(dateRestriction); - - Field lowerBound = when(validityDateRange.getStart().lessThan(restriction.getStart()), restriction.getStart()) - .otherwise(validityDateRange.getStart()); - - Field maxDate = getMinDateExpression(); - // we want to add +1 day to the end date - except when it's the max date already - Field restrictionUpperBound = when(restriction.getEnd().eq(maxDate), maxDate).otherwise(addDays(restriction.getEnd(), inline(1))); - Field upperBound = when(validityDateRange.getEnd().greaterThan(restriction.getEnd()), restrictionUpperBound) - .otherwise(validityDateRange.getEnd()); - - return ColumnDateRange.of(lowerBound, upperBound); - } - - private ColumnDateRange toColumnDateRange(CDateRange dateRestriction) { - - Field startDateExpression = getMinDateExpression(); - Field endDateExpression = getMaxDateExpression(); - - if (dateRestriction.hasLowerBound()) { - startDateExpression = inline(Date.valueOf(dateRestriction.getMin())); - } - if (dateRestriction.hasUpperBound()) { - endDateExpression = inline(Date.valueOf(dateRestriction.getMax())); - } - - return ColumnDateRange.of(startDateExpression, endDateExpression); - } - - @NotNull - @Override - public Collection> orderByValidityDates( - Function, ? extends SortField> ordering, - List> validityDateFields) { - - return List.of( - ordering.apply(nullif(validityDateFields.getFirst(), getMinDateExpression())).nullsLast(), - ordering.apply(nullif(validityDateFields.getLast(), getMaxDateExpression())).nullsLast() - ); - } - - @Override - public Field getMinDateExpression() { - return field("toDate32({0})", Date.class, MIN_DATE_VALUE); - } - - @Override - public Field getMaxDateExpression() { - return field("toDate32({0})", Date.class, MAX_DATE_VALUE); - } - - @Override - public ColumnDateRange forArbitraryDateRange(DaterangeSelectOrFilter daterangeSelectOrFilter) { - String tableName = daterangeSelectOrFilter.getTable().getName(); - if (daterangeSelectOrFilter.isSingleColumnDaterange()) { - Column column = daterangeSelectOrFilter.getColumn().resolve(); - return ofStartAndEnd(tableName, column, column); - } else { - return ofStartAndEnd(tableName, daterangeSelectOrFilter.getStartColumn().resolve(), daterangeSelectOrFilter.getEndColumn().resolve()); - } - } - - @Override - public ColumnDateRange aggregated(ColumnDateRange columnDateRange) { - return ColumnDateRange.of( - min(columnDateRange.getStart()), - max(columnDateRange.getEnd()) - ) - .as(columnDateRange.getAlias()); - } - - @Override - public ColumnDateRange toDualColumn(ColumnDateRange columnDateRange) { - return ColumnDateRange.of(columnDateRange.getStart(), columnDateRange.getEnd()); - } - - @Override - public ColumnDateRange intersection(ColumnDateRange left, ColumnDateRange right) { - Field greatest = DSL.greatest(left.getStart(), right.getStart()); - Field least = DSL.least(left.getEnd(), right.getEnd()); - return ColumnDateRange.of(greatest, least); - } - - @Override - public QueryStep unnestDaterange(ColumnDateRange nested, QueryStep predecessor, String cteName) { - return predecessor; - } - - @Override - public Field dateRangeAggregation(ColumnDateRange columnDateRange) { - return field("groupArray({0})", Object[].class, dateRangeToField(columnDateRange)); - } - - @Override - public Field dateRangeToField(ColumnDateRange columnDateRange) { - Field startDateExpression = field("{0}::Nullable(Integer)", Object.class, columnDateRange.getStart()); - Field endDateExpression = field("{0}::Nullable(Integer)", Object.class, columnDateRange.getEnd()); - - return function("tuple", Object.class, startDateExpression, endDateExpression); - } - - @Override - public Field cast(Field field, DataType type) { - if (type == SQLDataType.VARCHAR) { - return function("toString", type.getType(), field); - } - return function( - name("CAST"), - type.getType(), - field("{0} AS {1}", field, keyword(type.getName())) - ); - } - - @Override - public Field dateDistance(ChronoUnit datePart, Field startDate, Field endDate) { - - String unit = switch (datePart) { - case DAYS -> "days"; - case MONTHS -> "months"; - case YEARS, DECADES, CENTURIES -> "years"; - default -> throw new UnsupportedOperationException("Given ChronoUnit %s is not supported."); - }; - - Field dateDistance = function("age", Integer.class, inline(unit), startDate, endDate); - - // HANA does not support decades or centuries directly - dateDistance = switch (datePart) { - case DECADES -> dateDistance.divide(10); - case CENTURIES -> dateDistance.divide(100); - default -> dateDistance; - }; - - // otherwise HANA would return floating point numbers for date distances - return dateDistance.cast(Integer.class); - } - - - @Override - public Field random(Field column) { - return field( - "groupArraySample(1)({0})[1]", - column.getType(), - column - ); - } - - @Override - public Condition likeRegex(Field field, String pattern) { - return condition(function("match", Boolean.class, field, inline(pattern))); - } - - - @Override - public Field yearQuarter(Field dateField) { - return field("formatDateTime({0}, '%Y-Q%Q')", String.class, dateField); - } - - @Override - public ColumnDateRange allRangeIf(Condition condition) { - return ColumnDateRange.of( - when(condition.isTrue(), - getMinDateExpression() - ), - when(condition.isTrue(), - getMaxDateExpression() - ) - ); - } + public String getAnyCharRegex() { + return ANY_CHAR_REGEX; + } + + @Override + public Field asArrayRepr(List value) { + return array(value.toArray()); + } + + + @Override + public Field externalId(String id) { + // This is only used in CQExternal to define external Ids. + // Without explicitly tagging this value as Nullable, the Clickhouse QueryPlanner assumes that this it can never be null. + // But in the case of an anti-join (our Negation), this field will be sometimes null, while not testing as `is null` = true + return field("{0}::Nullable(String)", String.class, inline(id, String.class)); + } + + @Override + public Condition dateRestriction(ColumnDateRange dateRestriction, ColumnDateRange daterange) { + Condition dateRestrictionStartsBeforeDate = dateRestriction.getStart().lessThan(daterange.getEnd()); + Condition dateRestrictionEndsAfterDate = dateRestriction.getEnd().greaterThan(daterange.getStart()); + + return condition(dateRestrictionStartsBeforeDate.and(dateRestrictionEndsAfterDate)); + } + + + @Override + public ColumnDateRange forCDateRange(CDateRange daterange) { + + Field startDateExpression = getMinDateExpression(); + Field endDateExpression = getMaxDateExpression(); + + if (daterange.hasLowerBound()) { + startDateExpression = inline(Date.valueOf(daterange.getMin())); + } + if (daterange.hasUpperBound()) { + endDateExpression = inline(Date.valueOf(daterange.getMax().plusDays(1))); + } + + return ColumnDateRange.of(startDateExpression, endDateExpression); + } + + @Override + public Field toDateField(String dateExpression) { + return function( + "toDate", + Date.class, + inline(dateExpression), + inline(DEFAULT_DATE_FORMAT) + ); + } + + @Override + public Condition isNotEmptyDateRange(ColumnDateRange columnDateRange) { + Condition startNotMin = columnDateRange.getStart().notEqual(getMinDateExpression()); + Condition endNotMax = columnDateRange.getEnd().notEqual(getMaxDateExpression()); + return condition(startNotMin.and(endNotMax).neg()); + } + + @Override + public ColumnDateRange forValidityDate(ValidityDate validityDate) { + return toColumnDateRange(validityDate); + } + + + @Override + public ColumnDateRange allRange() { + return ColumnDateRange.of(getMinDateExpression(), getMaxDateExpression()); + } + + @Override + public Field anyValue(Field field) { + return DSL.anyValue(field); + } + + private ColumnDateRange toColumnDateRange(ValidityDate validityDate) { + + String tableName = validityDate.getConnector().resolveTableId().getTable(); + + Column startColumn; + Column endColumn; + + // if no end column is present, the only existing column is both start and end of the date range + if (validityDate.isSingleColumnDaterange()) { + Column column = validityDate.getColumn().resolve(); + startColumn = column; + endColumn = column; + } else { + startColumn = validityDate.getStartColumn().resolve(); + endColumn = validityDate.getEndColumn().resolve(); + } + + return ofStartAndEnd(tableName, startColumn, endColumn); + } + + + @Override + public Field arrayOut(List> fields) { + return field("arrayFilter(x -> x <> '', {0})", Object.class, array(fields)); + } + + private ColumnDateRange ofStartAndEnd(String tableName, Column startColumn, Column endColumn) { + + // Since coalesce makes Clickhouse certain, that the field is not nullable, it will do silly stuff with it down the line: + // missing values (for example in outer-joins) will be coerced to 0 = 01-01-1970, which is clearly not correct + // Therefore we tag the values as Nullable again to make Clickhouse show some respect + + Field rangeStart = field( + "{0}::Nullable(Date32)", + Date.class, + coalesce( + field(name(tableName, startColumn.getName()), Date.class), + getMinDateExpression() + ) + ); + // when aggregating date ranges, we want to treat the last day of the range as excluded, + // so when using the date value of the end column, we add +1 day as end of the date range + Field rangeEnd = field( + "{0}::Nullable(Date32)", + Date.class, + coalesce( + addDays(field(name(tableName, endColumn.getName()), Date.class), inline(1)), + getMaxDateExpression() + ) + ); + + return ColumnDateRange.of(rangeStart, rangeEnd); + } + + @Override + public Field addDays(Field dateColumn, Field amountOfDays) { + return function( + "addDays", + Date.class, + dateColumn, + amountOfDays + ); + } + + @Override + public ColumnDateRange forValidityDate(ValidityDate validityDate, CDateRange dateRestriction) { + + ColumnDateRange validityDateRange = toColumnDateRange(validityDate); + ColumnDateRange restriction = toColumnDateRange(dateRestriction); + + Field lowerBound = when( + validityDateRange.getStart().lessThan(restriction.getStart()), + restriction.getStart()).otherwise(validityDateRange.getStart()); + + Field maxDate = getMinDateExpression(); + // we want to add +1 day to the end date - except when it's the max date already + Field restrictionUpperBound = when(restriction.getEnd().eq(maxDate), maxDate).otherwise( + addDays(restriction.getEnd(), inline(1))); + Field upperBound = when( + validityDateRange.getEnd().greaterThan(restriction.getEnd()), + restrictionUpperBound).otherwise(validityDateRange.getEnd()); + + return ColumnDateRange.of(lowerBound, upperBound); + } + + private ColumnDateRange toColumnDateRange(CDateRange dateRestriction) { + + Field startDateExpression = getMinDateExpression(); + Field endDateExpression = getMaxDateExpression(); + + if (dateRestriction.hasLowerBound()) { + startDateExpression = inline(Date.valueOf(dateRestriction.getMin())); + } + if (dateRestriction.hasUpperBound()) { + endDateExpression = inline(Date.valueOf(dateRestriction.getMax())); + } + + return ColumnDateRange.of(startDateExpression, endDateExpression); + } + + @NotNull + @Override + public Collection> orderByValidityDates( + Function, ? extends SortField> ordering, + List> validityDateFields) { + + return List.of( + ordering.apply(nullif(validityDateFields.getFirst(), getMinDateExpression())).nullsLast(), + ordering.apply(nullif(validityDateFields.getLast(), getMaxDateExpression())).nullsLast() + ); + } + + @Override + public Field getMinDateExpression() { + return field("toDate32({0})", Date.class, MIN_DATE_VALUE); + } + + @Override + public Field getMaxDateExpression() { + return field("toDate32({0})", Date.class, MAX_DATE_VALUE); + } + + @Override + public ColumnDateRange forArbitraryDateRange(DaterangeSelectOrFilter daterangeSelectOrFilter) { + String tableName = daterangeSelectOrFilter.getTable().getName(); + if (daterangeSelectOrFilter.isSingleColumnDaterange()) { + Column column = daterangeSelectOrFilter.getColumn().resolve(); + return ofStartAndEnd(tableName, column, column); + } else { + return ofStartAndEnd( + tableName, + daterangeSelectOrFilter.getStartColumn().resolve(), + daterangeSelectOrFilter.getEndColumn().resolve()); + } + } + + @Override + public ColumnDateRange aggregated(ColumnDateRange columnDateRange) { + return ColumnDateRange.of( + min(columnDateRange.getStart()), + max(columnDateRange.getEnd()) + ).as(columnDateRange.getAlias()); + } + + @Override + public ColumnDateRange toDualColumn(ColumnDateRange columnDateRange) { + return ColumnDateRange.of(columnDateRange.getStart(), columnDateRange.getEnd()); + } + + @Override + public ColumnDateRange intersection(ColumnDateRange left, ColumnDateRange right) { + Field greatest = DSL.greatest(left.getStart(), right.getStart()); + Field least = DSL.least(left.getEnd(), right.getEnd()); + return ColumnDateRange.of(greatest, least); + } + + @Override + public QueryStep unnestDaterange(ColumnDateRange nested, QueryStep predecessor, String cteName) { + return predecessor; + } + + @Override + public Field dateRangeAggregation(ColumnDateRange columnDateRange) { + return field("groupArray({0})", Object[].class, dateRangeToField(columnDateRange)); + } + + @Override + public Field dateRangeToField(ColumnDateRange columnDateRange) { + Field startDateExpression = field("{0}::Nullable(Integer)", Object.class, columnDateRange.getStart()); + Field endDateExpression = field("{0}::Nullable(Integer)", Object.class, columnDateRange.getEnd()); + + return function("tuple", Object.class, startDateExpression, endDateExpression); + } + + @Override + public Field cast(Field field, DataType type) { + if (type == SQLDataType.VARCHAR) { + return function("toString", type.getType(), field); + } + return function( + name("CAST"), + type.getType(), + field("{0} AS {1}", field, keyword(type.getName())) + ); + } + + @Override + public Field dateDistance(ChronoUnit datePart, Field startDate, Field endDate) { + + String unit = switch (datePart) { + case DAYS -> "days"; + case MONTHS -> "months"; + case YEARS, DECADES, CENTURIES -> "years"; + default -> throw new UnsupportedOperationException("Given ChronoUnit %s is not supported."); + }; + + Field dateDistance = function("age", Integer.class, inline(unit), startDate, endDate); + + // HANA does not support decades or centuries directly + dateDistance = switch (datePart) { + case DECADES -> dateDistance.divide(10); + case CENTURIES -> dateDistance.divide(100); + default -> dateDistance; + }; + + // otherwise HANA would return floating point numbers for date distances + return dateDistance.cast(Integer.class); + } + + + @Override + public Field random(Field column) { + return field( + "groupArraySample(1)({0})[1]", + column.getType(), + column + ); + } + + @Override + public Condition likeRegex(Field field, String pattern) { + return condition(function("match", Boolean.class, field, inline(pattern))); + } + + + @Override + public Field yearQuarter(Field dateField) { + return field("formatDateTime({0}, '%Y-Q%Q')", String.class, dateField); + } + + @Override + public ColumnDateRange allRangeIf(Condition condition) { + return ColumnDateRange.of( + when(condition.isTrue(), getMinDateExpression() + ), + when(condition.isTrue(), getMaxDateExpression() + ) + ); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseResultSetProcessor.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseResultSetProcessor.java index 1abe7790866..cb0f9eddb57 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseResultSetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseResultSetProcessor.java @@ -40,8 +40,7 @@ public List getDateRange(ResultSet resultSet, int columnIndex) throws S int startDate; if (startDateExpression.equals(ClickhouseFunctionProvider.MIN_DATE_VALUE)) { startDate = CDateRange.NEGATIVE_INFINITY; - } - else { + } else { LocalDate dateValue = Date.valueOf(startDateExpression).toLocalDate(); startDate = CDate.ofLocalDate(dateValue); } @@ -50,8 +49,7 @@ public List getDateRange(ResultSet resultSet, int columnIndex) throws S int endDate; if (endDateExpression.equals(ClickhouseFunctionProvider.MAX_DATE_VALUE)) { endDate = CDateRange.POSITIVE_INFINITY; - } - else { + } else { LocalDate dateValue = Date.valueOf(endDateExpression).toLocalDate(); endDate = CDate.ofLocalDate(dateValue); } @@ -77,8 +75,7 @@ public List> toEpochDayRangeList(List raw) { } if (end == ClickhouseFunctionProvider.MAX_DATE_VALUE) { end = Integer.MAX_VALUE; - } - else { + } else { end--; } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseStratificationFunctions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseStratificationFunctions.java index 3126dcae204..c7ea740bc23 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseStratificationFunctions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseStratificationFunctions.java @@ -57,8 +57,8 @@ protected Field exclusiveUpper(ColumnDateRange dateRange) { @Override protected ColumnDateRange calcRange(Field start, Interval interval) { return ColumnDateRange.of( - calcStartDate(start, interval), - calcEndDate(start, interval) + calcStartDate(start, interval), + calcEndDate(start, interval) ); } @@ -84,10 +84,9 @@ public Field upperBoundYearEndQuarterAligned(ColumnDateRange dateRange) { Field yearEndQuarterAligned = addMonths(yearStartOfUpperBound, quartersInMonths); // we add +1 year to the quarter aligned end if it is less than the upper bound we want to align return when( - yearEndQuarterAligned.lessThan(dateRange.getEnd()), - shiftByInterval(yearEndQuarterAligned, Interval.ONE_YEAR_INTERVAL, inline(1), Offset.NONE) - ) - .otherwise(yearEndQuarterAligned); + yearEndQuarterAligned.lessThan(dateRange.getEnd()), + shiftByInterval(yearEndQuarterAligned, Interval.ONE_YEAR_INTERVAL, inline(1), Offset.NONE) + ).otherwise(yearEndQuarterAligned); } @Override @@ -132,7 +131,10 @@ public Field indexSelectorField(TemporalSamplerFactory indexSelector, Colu case LATEST -> max(inclusiveUpper(validityDate)); case RANDOM -> { // we calculate a random int which is in range of the date distance between upper and lower bound - Field dateDistanceInDays = functionProvider.dateDistance(ChronoUnit.DAYS, validityDate.getStart(), validityDate.getEnd()); + Field dateDistanceInDays = functionProvider.dateDistance( + ChronoUnit.DAYS, + validityDate.getStart(), + validityDate.getEnd()); Field randomAmountOfDays = function("RAND", Double.class).times(dateDistanceInDays); Field flooredAsInt = functionProvider.cast(floor(randomAmountOfDays), SQLDataType.INTEGER); // then we add this random amount (of days) to the start date @@ -147,10 +149,13 @@ public Field indexSelectorField(TemporalSamplerFactory indexSelector, Colu public Field shiftByInterval(Field startDate, Interval interval, Field amount, Offset offset) { Field multiplier = amount.plus(offset.getOffset()); return switch (interval) { - case ONE_YEAR_INTERVAL -> function("addYears", Date.class, startDate, multiplier.times(Interval.ONE_YEAR_INTERVAL.getAmount())); - case YEAR_AS_DAYS_INTERVAL -> addDays(startDate, multiplier.times(Interval.YEAR_AS_DAYS_INTERVAL.getAmount())); + case ONE_YEAR_INTERVAL -> + function("addYears", Date.class, startDate, multiplier.times(Interval.ONE_YEAR_INTERVAL.getAmount())); + case YEAR_AS_DAYS_INTERVAL -> + addDays(startDate, multiplier.times(Interval.YEAR_AS_DAYS_INTERVAL.getAmount())); case QUARTER_INTERVAL -> addMonths(startDate, multiplier.times(Interval.QUARTER_INTERVAL.getAmount())); - case NINETY_DAYS_INTERVAL -> addDays(startDate, multiplier.times(Interval.NINETY_DAYS_INTERVAL.getAmount())); + case NINETY_DAYS_INTERVAL -> + addDays(startDate, multiplier.times(Interval.NINETY_DAYS_INTERVAL.getAmount())); case ONE_DAY_INTERVAL -> addDays(startDate, multiplier.times(Interval.ONE_DAY_INTERVAL.getAmount())); }; } @@ -171,7 +176,8 @@ private Field getQuartersInMonths(Field date, Offset offset) { Field quarterExpression = functionProvider.yearQuarter(date); Field rightMostCharacter = function("RIGHT", String.class, quarterExpression, inline(1)); Field amountOfQuarters = functionProvider.cast(rightMostCharacter, SQLDataType.INTEGER) - .plus(offset.getOffset()); + .plus( + offset.getOffset()); return amountOfQuarters.times(MONTHS_PER_QUARTER); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaSqlFunctionProvider.java index cc2fdb4a7e3..777ff78eeb9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaSqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaSqlFunctionProvider.java @@ -79,7 +79,7 @@ public ColumnDateRange forCDateRange(CDateRange daterange) { // end date is expected to be handled as exclusive, but if it's already the maximum date, we can't add +1 day if (Objects.equals(daterange.getMax(), LocalDate.ofEpochDay(CDateRange.POSITIVE_INFINITY))) { throw new UnsupportedOperationException( - "Given daterange has an upper bound of CDateRange.POSITIVE_INFINITY, which is not supported by ConQuery's HANA dialect."); + "Given daterange has an upper bound of CDateRange.POSITIVE_INFINITY, which is not supported by ConQuery's HANA dialect."); } LocalDate exclusiveMaxDate = daterange.getMax().plusDays(1); endDateExpression = exclusiveMaxDate.toString(); @@ -91,24 +91,27 @@ public ColumnDateRange forCDateRange(CDateRange daterange) { @Override public Field toDateField(String dateExpression) { return function( - "TO_DATE", - Date.class, - inline(dateExpression), - inline(DEFAULT_DATE_FORMAT) + "TO_DATE", + Date.class, + inline(dateExpression), + inline(DEFAULT_DATE_FORMAT) ); } @Override public Condition isNotEmptyDateRange(ColumnDateRange columnDateRange) { - return columnDateRange.getStart().notEqual(getMinDateExpression()).or(columnDateRange.getEnd().notEqual(getMaxDateExpression())); + return columnDateRange.getStart() + .notEqual(getMinDateExpression()) + .or( + columnDateRange.getEnd().notEqual(getMaxDateExpression())); } @Override - public ColumnDateRange forValidityDate(ValidityDate validityDate) { - return toColumnDateRange(validityDate); - } + public ColumnDateRange forValidityDate(ValidityDate validityDate) { + return toColumnDateRange(validityDate); + } @Override public ColumnDateRange allRange() { @@ -116,12 +119,12 @@ public ColumnDateRange allRange() { } @Override - public Field anyValue(Field field) { - // Hana does not have any_value - return DSL.min(field); - } + public Field anyValue(Field field) { + // Hana does not have any_value + return DSL.min(field); + } - private ColumnDateRange toColumnDateRange(ValidityDate validityDate) { + private ColumnDateRange toColumnDateRange(ValidityDate validityDate) { String tableName = validityDate.getConnector().resolveTableId().getTable(); @@ -144,14 +147,14 @@ private ColumnDateRange toColumnDateRange(ValidityDate validityDate) { private ColumnDateRange ofStartAndEnd(String tableName, Column startColumn, Column endColumn) { Field rangeStart = coalesce( - field(name(tableName, startColumn.getName()), Date.class), - getMinDateExpression() + field(name(tableName, startColumn.getName()), Date.class), + getMinDateExpression() ); // when aggregating date ranges, we want to treat the last day of the range as excluded, // so when using the date value of the end column, we add +1 day as end of the date range Field rangeEnd = coalesce( - addDays(field(name(tableName, endColumn.getName()), Date.class), inline(1)), - getMaxDateExpression() + addDays(field(name(tableName, endColumn.getName()), Date.class), inline(1)), + getMaxDateExpression() ); return ColumnDateRange.of(rangeStart, rangeEnd); @@ -160,22 +163,20 @@ private ColumnDateRange ofStartAndEnd(String tableName, Column startColumn, Colu @Override public Field addDays(Field dateColumn, Field amountOfDays) { return function( - "ADD_DAYS", - Date.class, - dateColumn, - amountOfDays + "ADD_DAYS", + Date.class, + dateColumn, + amountOfDays ); } @Override public ColumnDateRange allRangeIf(Condition condition) { return ColumnDateRange.of( - when(condition.isTrue(), - getMinDateExpression() - ), - when(condition.isTrue(), - getMaxDateExpression() - ) + when(condition.isTrue(), getMinDateExpression() + ), + when(condition.isTrue(), getMaxDateExpression() + ) ); } @@ -186,13 +187,16 @@ public ColumnDateRange forValidityDate(ValidityDate validityDate, CDateRange dat ColumnDateRange validityDateRange = toColumnDateRange(validityDate); ColumnDateRange restriction = toColumnDateRange(dateRestriction); - Field lowerBound = when(validityDateRange.getStart().lessThan(restriction.getStart()), restriction.getStart()) - .otherwise(validityDateRange.getStart()); + Field lowerBound = when( + validityDateRange.getStart().lessThan(restriction.getStart()), + restriction.getStart()).otherwise(validityDateRange.getStart()); Field maxDate = toDateField(MAX_DATE_VALUE); // we want to add +1 day to the end date - except when it's the max date already - Field restrictionUpperBound = when(restriction.getEnd().eq(maxDate), maxDate).otherwise(addDays(restriction.getEnd(), inline(1))); - Field upperBound = when(validityDateRange.getEnd().greaterThan(restriction.getEnd()), restrictionUpperBound) - .otherwise(validityDateRange.getEnd()); + Field restrictionUpperBound = when(restriction.getEnd().eq(maxDate), maxDate).otherwise( + addDays(restriction.getEnd(), inline(1))); + Field upperBound = when( + validityDateRange.getEnd().greaterThan(restriction.getEnd()), + restrictionUpperBound).otherwise(validityDateRange.getEnd()); return ColumnDateRange.of(lowerBound, upperBound); } @@ -216,12 +220,12 @@ private ColumnDateRange toColumnDateRange(CDateRange dateRestriction) { @NotNull @Override public Collection> orderByValidityDates( - Function, ? extends SortField> ordering, - List> validityDateFields) { + Function, ? extends SortField> ordering, + List> validityDateFields) { return List.of( - ordering.apply(nullif(validityDateFields.getFirst(), getMinDateExpression())).nullsLast(), - ordering.apply(nullif(validityDateFields.getLast(), getMaxDateExpression())).nullsLast() + ordering.apply(nullif(validityDateFields.getFirst(), getMinDateExpression())).nullsLast(), + ordering.apply(nullif(validityDateFields.getLast(), getMaxDateExpression())).nullsLast() ); } @@ -242,17 +246,19 @@ public ColumnDateRange forArbitraryDateRange(DaterangeSelectOrFilter daterangeSe Column column = daterangeSelectOrFilter.getColumn().resolve(); return ofStartAndEnd(tableName, column, column); } else { - return ofStartAndEnd(tableName, daterangeSelectOrFilter.getStartColumn().resolve(), daterangeSelectOrFilter.getEndColumn().resolve()); + return ofStartAndEnd( + tableName, + daterangeSelectOrFilter.getStartColumn().resolve(), + daterangeSelectOrFilter.getEndColumn().resolve()); } } @Override public ColumnDateRange aggregated(ColumnDateRange columnDateRange) { return ColumnDateRange.of( - min(columnDateRange.getStart()), - max(columnDateRange.getEnd()) - ) - .as(columnDateRange.getAlias()); + min(columnDateRange.getStart()), + max(columnDateRange.getEnd()) + ).as(columnDateRange.getAlias()); } @Override @@ -278,10 +284,10 @@ public QueryStep unnestDaterange(ColumnDateRange nested, QueryStep predecessor, public Field dateRangeAggregation(ColumnDateRange columnDateRange) { Field stringAggregation = stringAggregation( - dateRangeToField(columnDateRange), - toChar(DATE_SET_SEPARATOR), - // The coalesce is necessary so that hana isn't upset about a potential `order by null` which happens for empty date-ranges - List.of(coalesce(columnDateRange.getStart(), getMinDateExpression())) + dateRangeToField(columnDateRange), + toChar(DATE_SET_SEPARATOR), + // The coalesce is necessary so that hana isn't upset about a potential `order by null` which happens for empty date-ranges + List.of(coalesce(columnDateRange.getStart(), getMinDateExpression())) ); // encapsulate all ranges (including empty ranges) within curly braces @@ -291,10 +297,12 @@ public Field dateRangeAggregation(ColumnDateRange columnDateRange) { @Override public Field dateRangeToField(ColumnDateRange columnDateRange) { // translation is handled in printer - return field("'[' || {0} || {2} || {1} || ')'", String.class, - cast(columnDateRange.getStart(), SQLDataType.VARCHAR), - cast(columnDateRange.getEnd(), SQLDataType.VARCHAR), - DATERANGE_SEPARATOR + return field( + "'[' || {0} || {2} || {1} || ')'", + String.class, + cast(columnDateRange.getStart(), SQLDataType.VARCHAR), + cast(columnDateRange.getEnd(), SQLDataType.VARCHAR), + DATERANGE_SEPARATOR ); } @@ -305,10 +313,10 @@ public Field cast(Field field, DataType type) { return function("TO_VARCHAR", type.getType(), field); } return function( - // Needs to be explicitly unquoted, otherwise Hana is angry when Jooq quotes it on occasion. - unquotedName("CAST"), - type.getType(), - field("{0} AS {1}", field, keyword(type.getName())) + // Needs to be explicitly unquoted, otherwise Hana is angry when Jooq quotes it on occasion. + unquotedName("CAST"), + type.getType(), + field("{0} AS {1}", field, keyword(type.getName())) ); } @@ -339,11 +347,11 @@ public Field dateDistance(ChronoUnit datePart, Field startDate, F @Override public Field random(Field column) { return field( - "{0}({1} {2})", - column.getType(), - keyword("FIRST_VALUE"), - column, - orderBy(function("RAND", Object.class)) + "{0}({1} {2})", + column.getType(), + keyword("FIRST_VALUE"), + column, + orderBy(function("RAND", Object.class)) ); } @@ -363,4 +371,4 @@ public Field isNull(Field field) { // DSl.isNull does not work in some cases for Hana. This accomplishes the same thing with extra steps (: return DSL.function("IFNULL", Boolean.class, field, inline(true)); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaStratificationFunctions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaStratificationFunctions.java index 2bae80d371e..cba64d0547c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaStratificationFunctions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/hana/HanaStratificationFunctions.java @@ -52,8 +52,8 @@ protected Field exclusiveUpper(ColumnDateRange dateRange) { @Override protected ColumnDateRange calcRange(Field start, Interval interval) { return ColumnDateRange.of( - calcStartDate(start, interval), - calcEndDate(start, interval) + calcStartDate(start, interval), + calcEndDate(start, interval) ); } @@ -70,11 +70,11 @@ public Field lowerBoundYearStart(ColumnDateRange dateRange) { @Override public Field upperBoundYearEnd(ColumnDateRange dateRange) { return DSL.field( - "SERIES_ROUND({0}, {1}, {2})", - Date.class, - dateRange.getEnd(), - DSL.inline("INTERVAL 1 YEAR"), - DSL.keyword("ROUND_UP") + "SERIES_ROUND({0}, {1}, {2})", + Date.class, + dateRange.getEnd(), + DSL.inline("INTERVAL 1 YEAR"), + DSL.keyword("ROUND_UP") ); } @@ -85,10 +85,9 @@ public Field upperBoundYearEndQuarterAligned(ColumnDateRange dateRange) { Field yearEndQuarterAligned = addMonths(yearStartOfUpperBound, quartersInMonths); // we add +1 year to the quarter aligned end if it is less than the upper bound we want to align return DSL.when( - yearEndQuarterAligned.lessThan(dateRange.getEnd()), - shiftByInterval(yearEndQuarterAligned, Interval.ONE_YEAR_INTERVAL, DSL.inline(1), Offset.NONE) - ) - .otherwise(yearEndQuarterAligned); + yearEndQuarterAligned.lessThan(dateRange.getEnd()), + shiftByInterval(yearEndQuarterAligned, Interval.ONE_YEAR_INTERVAL, DSL.inline(1), Offset.NONE) + ).otherwise(yearEndQuarterAligned); } @Override @@ -135,7 +134,10 @@ public Field indexSelectorField(TemporalSamplerFactory indexSelector, Colu case LATEST -> DSL.max(inclusiveUpper(validityDate)); case RANDOM -> { // we calculate a random int which is in range of the date distance between upper and lower bound - Field dateDistanceInDays = functionProvider.dateDistance(ChronoUnit.DAYS, validityDate.getStart(), validityDate.getEnd()); + Field dateDistanceInDays = functionProvider.dateDistance( + ChronoUnit.DAYS, + validityDate.getStart(), + validityDate.getEnd()); Field randomAmountOfDays = DSL.function("RAND", Double.class).times(dateDistanceInDays); Field flooredAsInt = functionProvider.cast(DSL.floor(randomAmountOfDays), SQLDataType.INTEGER); // then we add this random amount (of days) to the start date @@ -150,10 +152,17 @@ public Field indexSelectorField(TemporalSamplerFactory indexSelector, Colu public Field shiftByInterval(Field startDate, Interval interval, Field amount, Offset offset) { Field multiplier = amount.plus(offset.getOffset()); return switch (interval) { - case ONE_YEAR_INTERVAL -> DSL.function("ADD_YEARS", Date.class, startDate, multiplier.times(Interval.ONE_YEAR_INTERVAL.getAmount())); - case YEAR_AS_DAYS_INTERVAL -> addDays(startDate, multiplier.times(Interval.YEAR_AS_DAYS_INTERVAL.getAmount())); + case ONE_YEAR_INTERVAL -> + DSL.function( + "ADD_YEARS", + Date.class, + startDate, + multiplier.times(Interval.ONE_YEAR_INTERVAL.getAmount())); + case YEAR_AS_DAYS_INTERVAL -> + addDays(startDate, multiplier.times(Interval.YEAR_AS_DAYS_INTERVAL.getAmount())); case QUARTER_INTERVAL -> addMonths(startDate, multiplier.times(Interval.QUARTER_INTERVAL.getAmount())); - case NINETY_DAYS_INTERVAL -> addDays(startDate, multiplier.times(Interval.NINETY_DAYS_INTERVAL.getAmount())); + case NINETY_DAYS_INTERVAL -> + addDays(startDate, multiplier.times(Interval.NINETY_DAYS_INTERVAL.getAmount())); case ONE_DAY_INTERVAL -> addDays(startDate, multiplier.times(Interval.ONE_DAY_INTERVAL.getAmount())); }; } @@ -180,11 +189,11 @@ private Field calcDate(Field start, Interval interval, Offset offset private static Field jumpToYearStart(Field date) { return DSL.field( - "SERIES_ROUND({0}, {1}, {2})", - Date.class, - date, - DSL.inline("INTERVAL 1 YEAR"), - DSL.keyword("ROUND_DOWN") + "SERIES_ROUND({0}, {1}, {2})", + Date.class, + date, + DSL.inline("INTERVAL 1 YEAR"), + DSL.keyword("ROUND_DOWN") ); } @@ -192,7 +201,8 @@ private Field getQuartersInMonths(Field date, Offset offset) { Field quarterExpression = functionProvider.yearQuarter(date); Field rightMostCharacter = DSL.function("RIGHT", String.class, quarterExpression, DSL.inline(1)); Field amountOfQuarters = functionProvider.cast(rightMostCharacter, SQLDataType.INTEGER) - .plus(offset.getOffset()); + .plus( + offset.getOffset()); return amountOfQuarters.times(MONTHS_PER_QUARTER); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java index 190aa2c512a..53d68bbfbcf 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java @@ -32,14 +32,17 @@ class AbsoluteStratification { private final QueryStep baseStep; private final StratificationFunctions stratificationFunctions; - public QueryStep createStratificationTable(List resolutionAndAlignments, ConversionContext context) { + public QueryStep createStratificationTable( + List resolutionAndAlignments, + ConversionContext context) { QueryStep intSeriesStep = createIntSeriesStep(); QueryStep indexStartStep = createIndexStartStep(); List resolutionTables = resolutionAndAlignments.stream() - .map(resolutionAndAlignment -> createResolutionTable(indexStartStep, resolutionAndAlignment)) - .toList(); + .map( + resolutionAndAlignment -> createResolutionTable(indexStartStep, resolutionAndAlignment)) + .toList(); List predecessors = List.of(baseStep, intSeriesStep, indexStartStep); return StratificationTableFactory.unionResolutionTables(resolutionTables, predecessors, context); @@ -53,58 +56,72 @@ private QueryStep createIntSeriesStep() { FieldWrapper seriesIndex = new FieldWrapper<>(stratificationFunctions.intSeriesField()); - Selects selects = Selects.builder() - .ids(ids) - .sqlSelect(seriesIndex) - .build(); + Selects selects = Selects.builder().ids(ids).sqlSelect(seriesIndex).build(); - Table seriesTable = stratificationFunctions.generateIntSeries(INDEX_START, INDEX_END).asTable(SharedAliases.INDEX.getAlias()); + Table seriesTable = stratificationFunctions.generateIntSeries(INDEX_START, INDEX_END) + .asTable( + SharedAliases.INDEX.getAlias()); return QueryStep.builder() - .cteName(FormCteStep.INT_SERIES.getSuffix()) - .selects(selects) - .fromTable(seriesTable) - .build(); + .cteName(FormCteStep.INT_SERIES.getSuffix()) + .selects(selects) + .fromTable( + seriesTable) + .build(); } private QueryStep createIndexStartStep() { Selects baseStepSelects = baseStep.getQualifiedSelects(); - Preconditions.checkArgument(baseStepSelects.getStratificationDate().isPresent(), "The base step must have a stratification date set"); + Preconditions.checkArgument( + baseStepSelects.getStratificationDate().isPresent(), + "The base step must have a stratification date set"); ColumnDateRange bounds = baseStepSelects.getStratificationDate().get(); - Field indexStart = stratificationFunctions.absoluteIndexStartDate(bounds).as(SharedAliases.INDEX_START.getAlias()); - Field yearStart = stratificationFunctions.lowerBoundYearStart(bounds).as(SharedAliases.YEAR_START.getAlias()); + Field indexStart = stratificationFunctions.absoluteIndexStartDate(bounds) + .as( + SharedAliases.INDEX_START.getAlias()); + Field yearStart = stratificationFunctions.lowerBoundYearStart(bounds) + .as( + SharedAliases.YEAR_START.getAlias()); Field yearEnd = stratificationFunctions.upperBoundYearEnd(bounds).as(SharedAliases.YEAR_END.getAlias()); - Field yearEndQuarterAligned = stratificationFunctions.upperBoundYearEndQuarterAligned(bounds).as(SharedAliases.YEAR_END_QUARTER_ALIGNED.getAlias()); - Field quarterStart = stratificationFunctions.lowerBoundQuarterStart(bounds).as(SharedAliases.QUARTER_START.getAlias()); - Field quarterEnd = stratificationFunctions.upperBoundQuarterEnd(bounds).as(SharedAliases.QUARTER_END.getAlias()); + Field yearEndQuarterAligned = stratificationFunctions.upperBoundYearEndQuarterAligned(bounds) + .as( + SharedAliases.YEAR_END_QUARTER_ALIGNED.getAlias()); + Field quarterStart = stratificationFunctions.lowerBoundQuarterStart(bounds) + .as( + SharedAliases.QUARTER_START.getAlias()); + Field quarterEnd = stratificationFunctions.upperBoundQuarterEnd(bounds) + .as( + SharedAliases.QUARTER_END.getAlias()); List> startDates = Stream.of( - indexStart, - yearStart, - yearEnd, - yearEndQuarterAligned, - quarterStart, - quarterEnd - ) - .map(FieldWrapper::new) - .toList(); + indexStart, + yearStart, + yearEnd, + yearEndQuarterAligned, + quarterStart, + quarterEnd + ).map(FieldWrapper::new).toList(); Selects selects = Selects.builder() - .ids(baseStepSelects.getIds()) - .stratificationDate(Optional.of(bounds)) - .sqlSelects(startDates) - .build(); + .ids(baseStepSelects.getIds()) + .stratificationDate( + Optional.of(bounds)) + .sqlSelects(startDates) + .build(); return QueryStep.builder() - .cteName(FormCteStep.INDEX_START.getSuffix()) - .selects(selects) - .fromTable(QueryStep.toTableLike(baseStep.getCteName())) - .build(); + .cteName(FormCteStep.INDEX_START.getSuffix()) + .selects(selects) + .fromTable( + QueryStep.toTableLike(baseStep.getCteName())) + .build(); } - private QueryStep createResolutionTable(QueryStep indexStartStep, ExportForm.ResolutionAndAlignment resolutionAndAlignment) { + private QueryStep createResolutionTable( + QueryStep indexStartStep, + ExportForm.ResolutionAndAlignment resolutionAndAlignment) { return switch (resolutionAndAlignment.getResolution()) { case COMPLETE -> createCompleteTable(); case YEARS, QUARTERS, DAYS -> createIntervalTable(indexStartStep, resolutionAndAlignment); @@ -122,68 +139,85 @@ private QueryStep createCompleteTable() { ColumnDateRange completeRange = baseStepSelects.getStratificationDate().get(); - Selects selects = Selects.builder() - .ids(ids) - .stratificationDate(Optional.of(completeRange)) - .build(); + Selects selects = Selects.builder().ids(ids).stratificationDate(Optional.of(completeRange)).build(); return QueryStep.builder() - .cteName(FormCteStep.COMPLETE.getSuffix()) - .selects(selects) - .fromTable(QueryStep.toTableLike(baseStep.getCteName())) - .build(); + .cteName(FormCteStep.COMPLETE.getSuffix()) + .selects(selects) + .fromTable( + QueryStep.toTableLike(baseStep.getCteName())) + .build(); } - private QueryStep createIntervalTable(QueryStep indexStartStep, ExportForm.ResolutionAndAlignment resolutionAndAlignment) { + private QueryStep createIntervalTable( + QueryStep indexStartStep, + ExportForm.ResolutionAndAlignment resolutionAndAlignment) { QueryStep countsCte = createCountsCte(indexStartStep, resolutionAndAlignment); - Preconditions.checkArgument(countsCte.getSelects().getStratificationDate().isPresent(), "The countsCte must have a stratification date set"); + Preconditions.checkArgument( + countsCte.getSelects().getStratificationDate().isPresent(), + "The countsCte must have a stratification date set"); Selects countsCteSelects = countsCte.getQualifiedSelects(); ColumnDateRange stratificationRange = stratificationFunctions.createStratificationRange( - resolutionAndAlignment, - countsCteSelects.getStratificationDate().get() + resolutionAndAlignment, + countsCteSelects.getStratificationDate().get() ); - Field index = stratificationFunctions.index(countsCteSelects.getIds(), countsCte.getQualifiedSelects().getStratificationDate()); - SqlIdColumns ids = countsCteSelects.getIds().withAbsoluteStratification(resolutionAndAlignment.getResolution(), index); + Field index = stratificationFunctions.index( + countsCteSelects.getIds(), + countsCte.getQualifiedSelects().getStratificationDate()); + SqlIdColumns ids = countsCteSelects.getIds() + .withAbsoluteStratification( + resolutionAndAlignment.getResolution(), + index); Selects selects = Selects.builder() - .ids(ids) - .stratificationDate(Optional.ofNullable(stratificationRange)) - .build(); + .ids(ids) + .stratificationDate( + Optional.ofNullable(stratificationRange)) + .build(); - Condition stopOnMaxResolutionWindowCount = stratificationFunctions.stopOnMaxResolutionWindowCount(resolutionAndAlignment); + Condition stopOnMaxResolutionWindowCount = stratificationFunctions.stopOnMaxResolutionWindowCount( + resolutionAndAlignment); return QueryStep.builder() - .cteName(FormCteStep.stratificationCte(resolutionAndAlignment.getResolution()).getSuffix()) - .selects(selects) - .fromTable(QueryStep.toTableLike(countsCte.getCteName())) - .fromTable(QueryStep.toTableLike(FormCteStep.INT_SERIES.getSuffix())) - .conditions(List.of(stopOnMaxResolutionWindowCount)) - .predecessor(countsCte) - .build(); + .cteName( + FormCteStep.stratificationCte(resolutionAndAlignment.getResolution()).getSuffix()) + .selects( + selects) + .fromTable(QueryStep.toTableLike(countsCte.getCteName())) + .fromTable( + QueryStep.toTableLike(FormCteStep.INT_SERIES.getSuffix())) + .conditions( + List.of(stopOnMaxResolutionWindowCount)) + .predecessor(countsCte) + .build(); } - private QueryStep createCountsCte(QueryStep indexStartStep, ExportForm.ResolutionAndAlignment resolutionAndAlignment) { + private QueryStep createCountsCte( + QueryStep indexStartStep, + ExportForm.ResolutionAndAlignment resolutionAndAlignment) { Selects indexStartSelects = indexStartStep.getQualifiedSelects(); - Preconditions.checkArgument(indexStartSelects.getStratificationDate().isPresent(), "The indexStartStep must have a stratification date set"); + Preconditions.checkArgument( + indexStartSelects.getStratificationDate().isPresent(), + "The indexStartStep must have a stratification date set"); Field resolutionWindowCount = stratificationFunctions.calculateResolutionWindowCount( - resolutionAndAlignment, - indexStartSelects.getStratificationDate().get() + resolutionAndAlignment, + indexStartSelects.getStratificationDate().get() ); - Selects selects = indexStartSelects.toBuilder() - .sqlSelect(new FieldWrapper<>(resolutionWindowCount)) - .build(); + Selects selects = indexStartSelects.toBuilder().sqlSelect(new FieldWrapper<>(resolutionWindowCount)).build(); return QueryStep.builder() - .cteName(FormCteStep.countsCte(resolutionAndAlignment.getResolution()).getSuffix()) - .selects(selects) - .fromTable(QueryStep.toTableLike(indexStartStep.getCteName())) - .build(); + .cteName( + FormCteStep.countsCte(resolutionAndAlignment.getResolution()).getSuffix()) + .selects(selects) + .fromTable( + QueryStep.toTableLike(indexStartStep.getCteName())) + .build(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/CombinationNotSupportedException.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/CombinationNotSupportedException.java index a29f4999c51..a744a33b441 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/CombinationNotSupportedException.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/CombinationNotSupportedException.java @@ -8,10 +8,11 @@ class CombinationNotSupportedException extends RuntimeException { public CombinationNotSupportedException(ExportForm.ResolutionAndAlignment resolutionAndAlignment) { - super("Alignment %s does not fit the resolution %s.".formatted( + super( + "Alignment %s does not fit the resolution %s.".formatted( resolutionAndAlignment.getAlignment(), resolutionAndAlignment.getResolution() - )); + )); } public CombinationNotSupportedException(IndexPlacement indexPlacement, CalendarUnit timeUnit) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormConstants.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormConstants.java index e51fe3e87f9..f66ea3dc077 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormConstants.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormConstants.java @@ -27,12 +27,16 @@ public class FormConstants { /** * The index date from which we start when calculating a feature range. */ - public static Field INDEX_START_NEGATIVE = DSL.field(DSL.name(SharedAliases.INDEX_START_NEGATIVE.getAlias()), Date.class); + public static Field INDEX_START_NEGATIVE = DSL.field( + DSL.name(SharedAliases.INDEX_START_NEGATIVE.getAlias()), + Date.class); /** * The index date from which we start when calculating an outcome range. */ - public static Field INDEX_START_POSITIVE = DSL.field(DSL.name(SharedAliases.INDEX_START_POSITIVE.getAlias()), Date.class); + public static Field INDEX_START_POSITIVE = DSL.field( + DSL.name(SharedAliases.INDEX_START_POSITIVE.getAlias()), + Date.class); /** * The quarter start of the lower bound of an absolute stratification range. The stratification range this date is referring to can be an @@ -67,26 +71,36 @@ public class FormConstants { * date is referring to can be an {@link AbsoluteFormQuery#getDateRange()} or for an {@link EntityDateQuery} the respective entities date range bound by * the {@link EntityDateQuery#getDateRange()}. */ - public static Field YEAR_END_QUARTER_ALIGNED = DSL.field(DSL.name(SharedAliases.YEAR_END_QUARTER_ALIGNED.getAlias()), Date.class); + public static Field YEAR_END_QUARTER_ALIGNED = DSL.field( + DSL.name(SharedAliases.YEAR_END_QUARTER_ALIGNED.getAlias()), + Date.class); /** * The amount times an interval added to a date, aligned on a year. */ - public static Field YEAR_ALIGNED_COUNT = DSL.field(DSL.name(SharedAliases.YEAR_ALIGNED_COUNT.getAlias()), Integer.class); + public static Field YEAR_ALIGNED_COUNT = DSL.field( + DSL.name(SharedAliases.YEAR_ALIGNED_COUNT.getAlias()), + Integer.class); /** * The amount times an interval added to a date, aligned on a quarter. */ - public static Field QUARTER_ALIGNED_COUNT = DSL.field(DSL.name(SharedAliases.QUARTER_ALIGNED_COUNT.getAlias()), Integer.class); + public static Field QUARTER_ALIGNED_COUNT = DSL.field( + DSL.name(SharedAliases.QUARTER_ALIGNED_COUNT.getAlias()), + Integer.class); /** * The amount times an interval added to a date, aligned a day. */ - public static Field DAY_ALIGNED_COUNT = DSL.field(DSL.name(SharedAliases.DAY_ALIGNED_COUNT.getAlias()), Integer.class); + public static Field DAY_ALIGNED_COUNT = DSL.field( + DSL.name(SharedAliases.DAY_ALIGNED_COUNT.getAlias()), + Integer.class); /** * The int field generated by a generate_series SQL set-returning function. */ - public static Field SERIES_INDEX = DSL.field(DSL.name(SharedAliases.SERIES_INDEX.getAlias()), Integer.class); + public static Field SERIES_INDEX = DSL.field( + DSL.name(SharedAliases.SERIES_INDEX.getAlias()), + Integer.class); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormCteStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormCteStep.java index 7dbd2baedb1..f252b8d5c30 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormCteStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormCteStep.java @@ -13,31 +13,22 @@ public enum FormCteStep implements CteStep { EXTRACT_IDS("extract_ids"), // entity date - UNNEST_ENTITY_DATE_CTE("unnest_entity_date"), - OVERWRITE_BOUNDS("overwrite_bounds"), + UNNEST_ENTITY_DATE_CTE("unnest_entity_date"), OVERWRITE_BOUNDS("overwrite_bounds"), // relative form - UNNEST_DATES("unnest_dates"), - INDEX_SELECTOR("index_selector"), - TOTAL_BOUNDS("total_bounds"), + UNNEST_DATES("unnest_dates"), INDEX_SELECTOR("index_selector"), TOTAL_BOUNDS("total_bounds"), // stratification - INDEX_START("index_start"), - INT_SERIES("int_series"), - YEAR_COUNTS("year_counts"), - QUARTER_COUNTS("quarter_counts"), - DAY_COUNTS("day_counts"), - DAYS("days"), - QUARTERS("quarters"), - YEARS("years"), - COMPLETE("complete"), - FULL_STRATIFICATION("full_stratification"); + INDEX_START("index_start"), INT_SERIES("int_series"), YEAR_COUNTS("year_counts"), QUARTER_COUNTS( + "quarter_counts"), DAY_COUNTS("day_counts"), DAYS("days"), QUARTERS("quarters"), YEARS("years"), COMPLETE( + "complete"), FULL_STRATIFICATION("full_stratification"); private final String suffix; public static FormCteStep countsCte(Resolution resolution) { return switch (resolution) { - case COMPLETE -> throw new UnsupportedOperationException("COMPLETE resolution does not require a counts CTE"); + case COMPLETE -> + throw new UnsupportedOperationException("COMPLETE resolution does not require a counts CTE"); case YEARS -> YEAR_COUNTS; case QUARTERS -> QUARTER_COUNTS; case DAYS -> DAY_COUNTS; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormType.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormType.java index d448a6ba339..9734764097b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormType.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormType.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.sql.conversion.forms; public enum FormType { - ABSOLUTE, - ENTITY_DATE, - RELATIVE + ABSOLUTE, ENTITY_DATE, RELATIVE } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/Offset.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/Offset.java index 92242c63bf2..fe5f0e1225f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/Offset.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/Offset.java @@ -12,8 +12,7 @@ @Getter @RequiredArgsConstructor public enum Offset { - MINUS_ONE(-1), - NONE(0), - ONE(1); + MINUS_ONE(-1), NONE(0), ONE(1); + private final int offset; } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java index d06755cbdce..cf42ff1d02f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java @@ -47,18 +47,26 @@ public QueryStep createRelativeStratificationTable(RelativeFormQuery form, Conve // we want to create the stratification for each distinct validity date range of an entity, // so we first need to unnest the validity date in case it is a multirange - Preconditions.checkArgument(baseStep.getSelects().getValidityDate().isPresent(), "Base step must contain a validity date"); + Preconditions.checkArgument( + baseStep.getSelects().getValidityDate().isPresent(), + "Base step must contain a validity date"); String unnestCteName = FormCteStep.UNNEST_DATES.getSuffix(); - QueryStep withUnnestedValidityDate = functionProvider.unnestDaterange(baseStep.getSelects().getValidityDate().get(), baseStep, unnestCteName); + QueryStep withUnnestedValidityDate = functionProvider.unnestDaterange( + baseStep.getSelects().getValidityDate().get(), + baseStep, + unnestCteName); QueryStep indexSelectorStep = createIndexSelectorStep(form, withUnnestedValidityDate); QueryStep indexStartStep = createIndexStartStep(form, indexSelectorStep); QueryStep totalBoundsStep = createTotalBoundsStep(form, indexStartStep); - List tables = form.getResolutionsAndAlignmentMap().stream() - .map(ExportForm.ResolutionAndAlignment::getResolution) - .map(resolution -> createResolutionTable(totalBoundsStep, resolution, form, context)) - .toList(); + List tables = form.getResolutionsAndAlignmentMap() + .stream() + .map( + ExportForm.ResolutionAndAlignment::getResolution) + .map( + resolution -> createResolutionTable(totalBoundsStep, resolution, form, context)) + .toList(); List predecessors = new ArrayList<>(); predecessors.add(baseStep); @@ -77,21 +85,25 @@ private QueryStep createIndexSelectorStep(RelativeFormQuery form, QueryStep prer Selects predecessorSelects = prerequisite.getQualifiedSelects(); ColumnDateRange validityDate = predecessorSelects.getValidityDate() - .orElseThrow(() -> new IllegalStateException("Expecting a validity date to be present")); + .orElseThrow( + () -> new IllegalStateException("Expecting a validity date to be present")); Field indexDate = stratificationFunctions.indexSelectorField(form.getIndexSelector(), validityDate) - .as(SharedAliases.INDEX_SELECTOR.getAlias()); + .as( + SharedAliases.INDEX_SELECTOR.getAlias()); Selects selects = Selects.builder() - .ids(predecessorSelects.getIds()) - .sqlSelect(new FieldWrapper<>(indexDate)) - .build(); + .ids(predecessorSelects.getIds()) + .sqlSelect( + new FieldWrapper<>(indexDate)) + .build(); return QueryStep.builder() - .cteName(FormCteStep.INDEX_SELECTOR.getSuffix()) - .selects(selects) - .fromTable(QueryStep.toTableLike(prerequisite.getCteName())) - .groupBy(predecessorSelects.getIds().toFields()) - .build(); + .cteName(FormCteStep.INDEX_SELECTOR.getSuffix()) + .selects(selects) + .fromTable( + QueryStep.toTableLike(prerequisite.getCteName())) + .groupBy(predecessorSelects.getIds().toFields()) + .build(); } /** @@ -100,21 +112,19 @@ private QueryStep createIndexSelectorStep(RelativeFormQuery form, QueryStep prer */ private QueryStep createIndexStartStep(RelativeFormQuery form, QueryStep indexSelectorStep) { - List> indexStartFields = stratificationFunctions.indexStartFields(form.getIndexPlacement(), form.getTimeUnit()).stream() - .map(FieldWrapper::new) - .toList(); + List> indexStartFields = stratificationFunctions.indexStartFields( + form.getIndexPlacement(), + form.getTimeUnit()).stream().map(FieldWrapper::new).toList(); // add index start fields to qualified selects of previous step - Selects selects = indexSelectorStep.getQualifiedSelects() - .toBuilder() - .sqlSelects(indexStartFields) - .build(); + Selects selects = indexSelectorStep.getQualifiedSelects().toBuilder().sqlSelects(indexStartFields).build(); return QueryStep.builder() - .cteName(FormCteStep.INDEX_START.getSuffix()) - .selects(selects) - .fromTable(QueryStep.toTableLike(indexSelectorStep.getCteName())) - .build(); + .cteName(FormCteStep.INDEX_START.getSuffix()) + .selects(selects) + .fromTable( + QueryStep.toTableLike(indexSelectorStep.getCteName())) + .build(); } /** @@ -125,67 +135,126 @@ private QueryStep createTotalBoundsStep(RelativeFormQuery form, QueryStep indexS Interval interval = getInterval(form.getTimeUnit(), Resolution.COMPLETE); Range intRange = toGenerateSeriesBounds(form, Resolution.COMPLETE); - Field minStratificationDate = stratificationFunctions.shiftByInterval(INDEX_START_NEGATIVE, interval, DSL.inline(intRange.getMin()), Offset.NONE); - Field maxStratificationDate = stratificationFunctions.shiftByInterval(INDEX_START_POSITIVE, interval, DSL.inline(intRange.getMax()), Offset.NONE); - ColumnDateRange minAndMaxStratificationDate = stratificationFunctions.ofStartAndEnd(minStratificationDate, maxStratificationDate) - .as(SharedAliases.STRATIFICATION_BOUNDS.getAlias()); + Field minStratificationDate = stratificationFunctions.shiftByInterval( + INDEX_START_NEGATIVE, + interval, + DSL.inline(intRange.getMin()), + Offset.NONE); + Field maxStratificationDate = stratificationFunctions.shiftByInterval( + INDEX_START_POSITIVE, + interval, + DSL.inline(intRange.getMax()), + Offset.NONE); + ColumnDateRange minAndMaxStratificationDate = stratificationFunctions.ofStartAndEnd( + minStratificationDate, + maxStratificationDate).as(SharedAliases.STRATIFICATION_BOUNDS.getAlias()); // add min and max stratification date to qualified selects of previous step Selects selects = indexStartStep.getQualifiedSelects() - .toBuilder() - .stratificationDate(Optional.of(minAndMaxStratificationDate)) - .build(); + .toBuilder() + .stratificationDate( + Optional.of(minAndMaxStratificationDate)) + .build(); return QueryStep.builder() - .cteName(FormCteStep.TOTAL_BOUNDS.getSuffix()) - .selects(selects) - .fromTable(QueryStep.toTableLike(indexStartStep.getCteName())) - .build(); + .cteName(FormCteStep.TOTAL_BOUNDS.getSuffix()) + .selects(selects) + .fromTable( + QueryStep.toTableLike(indexStartStep.getCteName())) + .build(); } - private QueryStep createResolutionTable(QueryStep indexStartStep, Resolution resolution, RelativeFormQuery form, ConversionContext context) { + private QueryStep createResolutionTable( + QueryStep indexStartStep, + Resolution resolution, + RelativeFormQuery form, + ConversionContext context) { return switch (resolution) { case COMPLETE -> createCompleteTable(indexStartStep, form, context); case YEARS, QUARTERS, DAYS -> createIntervalTable(indexStartStep, resolution, form, context); }; } - private QueryStep createCompleteTable(QueryStep totalBoundsStep, RelativeFormQuery form, ConversionContext context) { + private QueryStep createCompleteTable( + QueryStep totalBoundsStep, + RelativeFormQuery form, + ConversionContext context) { Selects predecessorSelects = totalBoundsStep.getQualifiedSelects(); Interval interval = getInterval(form.getTimeUnit(), Resolution.COMPLETE); Range intRange = toGenerateSeriesBounds(form, Resolution.COMPLETE); - QueryStep featureTable = form.getTimeCountBefore() > 0 ? createCompleteFeatureTable(predecessorSelects, interval, intRange, totalBoundsStep) : null; - QueryStep outcomeTable = form.getTimeCountAfter() > 0 ? createCompleteOutcomeTable(predecessorSelects, interval, intRange, totalBoundsStep) : null; + QueryStep featureTable = form.getTimeCountBefore() > 0 ? createCompleteFeatureTable( + predecessorSelects, + interval, + intRange, + totalBoundsStep) : null; + QueryStep outcomeTable = form.getTimeCountAfter() > 0 ? createCompleteOutcomeTable( + predecessorSelects, + interval, + intRange, + totalBoundsStep) : null; return QueryStep.createUnionAllStep( - Stream.concat(Stream.ofNullable(outcomeTable), Stream.ofNullable(featureTable)).toList(), - FormCteStep.COMPLETE.getSuffix(), - Collections.emptyList(), - context.isNegation(), functionProvider + Stream.concat(Stream.ofNullable(outcomeTable), Stream.ofNullable(featureTable)).toList(), + FormCteStep.COMPLETE.getSuffix(), + Collections.emptyList(), + context.isNegation(), + functionProvider ); } - private QueryStep createCompleteFeatureTable(Selects predecessorSelects, Interval interval, Range intRange, QueryStep totalBoundsStep) { + private QueryStep createCompleteFeatureTable( + Selects predecessorSelects, + Interval interval, + Range intRange, + QueryStep totalBoundsStep) { Field featureIndex = DSL.field(DSL.inline(-1)).as(SharedAliases.INDEX.getAlias()); - SqlIdColumns featureIds = predecessorSelects.getIds().withRelativeStratification(Resolution.COMPLETE, featureIndex, INDEX_SELECTOR); - Field rangeStart = stratificationFunctions.shiftByInterval(INDEX_START_NEGATIVE, interval, DSL.inline(intRange.getMin()), Offset.NONE); + SqlIdColumns featureIds = predecessorSelects.getIds() + .withRelativeStratification( + Resolution.COMPLETE, + featureIndex, + INDEX_SELECTOR); + Field rangeStart = stratificationFunctions.shiftByInterval( + INDEX_START_NEGATIVE, + interval, + DSL.inline(intRange.getMin()), + Offset.NONE); return createIntervalStep(featureIds, rangeStart, INDEX_START_NEGATIVE, Optional.empty(), totalBoundsStep); } - private QueryStep createCompleteOutcomeTable(Selects predecessorSelects, Interval interval, Range intRange, QueryStep totalBoundsStep) { + private QueryStep createCompleteOutcomeTable( + Selects predecessorSelects, + Interval interval, + Range intRange, + QueryStep totalBoundsStep) { Field outcomeIndex = DSL.field(DSL.inline(1)).as(SharedAliases.INDEX.getAlias()); - SqlIdColumns outcomeIds = predecessorSelects.getIds().withRelativeStratification(Resolution.COMPLETE, outcomeIndex, INDEX_SELECTOR); - Field rangeEnd = stratificationFunctions.shiftByInterval(INDEX_START_POSITIVE, interval, DSL.inline(intRange.getMax()), Offset.NONE); + SqlIdColumns outcomeIds = predecessorSelects.getIds() + .withRelativeStratification( + Resolution.COMPLETE, + outcomeIndex, + INDEX_SELECTOR); + Field rangeEnd = stratificationFunctions.shiftByInterval( + INDEX_START_POSITIVE, + interval, + DSL.inline(intRange.getMax()), + Offset.NONE); return createIntervalStep(outcomeIds, INDEX_START_POSITIVE, rangeEnd, Optional.empty(), totalBoundsStep); } - private QueryStep createIntervalTable(QueryStep totalBoundsStep, Resolution resolution, RelativeFormQuery form, ConversionContext context) { + private QueryStep createIntervalTable( + QueryStep totalBoundsStep, + Resolution resolution, + RelativeFormQuery form, + ConversionContext context) { Field seriesIndex = stratificationFunctions.intSeriesField(); Selects predecessorSelects = totalBoundsStep.getQualifiedSelects(); - SqlIdColumns ids = predecessorSelects.getIds().withRelativeStratification(resolution, seriesIndex, INDEX_SELECTOR); + SqlIdColumns ids = predecessorSelects.getIds() + .withRelativeStratification( + resolution, + seriesIndex, + INDEX_SELECTOR); Interval interval = getInterval(form.getTimeUnit(), resolution); Range bounds = toGenerateSeriesBounds(form, resolution); @@ -193,52 +262,80 @@ private QueryStep createIntervalTable(QueryStep totalBoundsStep, Resolution reso QueryStep timeAfterStep = createOutcomeTable(totalBoundsStep, interval, seriesIndex, bounds, ids); return QueryStep.createUnionAllStep( - List.of(timeBeforeStep, timeAfterStep), - FormCteStep.stratificationCte(resolution).getSuffix(), - Collections.emptyList(), - context.isNegation(), functionProvider + List.of(timeBeforeStep, timeAfterStep), + FormCteStep.stratificationCte(resolution).getSuffix(), + Collections.emptyList(), + context.isNegation(), + functionProvider ); } - private QueryStep createOutcomeTable(QueryStep totalBoundsStep, Interval interval, Field seriesIndex, Range bounds, SqlIdColumns ids) { - Field outcomeRangeStart = stratificationFunctions.shiftByInterval(INDEX_START_POSITIVE, interval, seriesIndex, Offset.MINUS_ONE); - Field outcomeRangeEnd = stratificationFunctions.shiftByInterval(INDEX_START_POSITIVE, interval, seriesIndex, Offset.NONE); - Table outcomeSeries = stratificationFunctions.generateIntSeries(1, bounds.getMax()).as(SharedAliases.INDEX.getAlias()); + private QueryStep createOutcomeTable( + QueryStep totalBoundsStep, + Interval interval, + Field seriesIndex, + Range bounds, + SqlIdColumns ids) { + Field outcomeRangeStart = stratificationFunctions.shiftByInterval( + INDEX_START_POSITIVE, + interval, + seriesIndex, + Offset.MINUS_ONE); + Field outcomeRangeEnd = stratificationFunctions.shiftByInterval( + INDEX_START_POSITIVE, + interval, + seriesIndex, + Offset.NONE); + Table outcomeSeries = stratificationFunctions.generateIntSeries(1, bounds.getMax()) + .as( + SharedAliases.INDEX.getAlias()); return createIntervalStep(ids, outcomeRangeStart, outcomeRangeEnd, Optional.of(outcomeSeries), totalBoundsStep); } - private QueryStep createFeatureTable(QueryStep totalBoundsStep, Interval interval, Field seriesIndex, Range bounds, SqlIdColumns ids) { - Field featureRangeStart = stratificationFunctions.shiftByInterval(INDEX_START_NEGATIVE, interval, seriesIndex, Offset.NONE); - Field featureRangeEnd = stratificationFunctions.shiftByInterval(INDEX_START_NEGATIVE, interval, seriesIndex, Offset.ONE); - Table featureSeries = stratificationFunctions.generateIntSeries(bounds.getMin(), -1).as(SharedAliases.INDEX.getAlias()); + private QueryStep createFeatureTable( + QueryStep totalBoundsStep, + Interval interval, + Field seriesIndex, + Range bounds, + SqlIdColumns ids) { + Field featureRangeStart = stratificationFunctions.shiftByInterval( + INDEX_START_NEGATIVE, + interval, + seriesIndex, + Offset.NONE); + Field featureRangeEnd = stratificationFunctions.shiftByInterval( + INDEX_START_NEGATIVE, + interval, + seriesIndex, + Offset.ONE); + Table featureSeries = stratificationFunctions.generateIntSeries(bounds.getMin(), -1) + .as( + SharedAliases.INDEX.getAlias()); return createIntervalStep(ids, featureRangeStart, featureRangeEnd, Optional.of(featureSeries), totalBoundsStep); } private QueryStep createIntervalStep( - SqlIdColumns ids, - Field rangeStart, - Field rangeEnd, - Optional> seriesTable, - QueryStep predecessor + SqlIdColumns ids, + Field rangeStart, + Field rangeEnd, + Optional> seriesTable, + QueryStep predecessor ) { Preconditions.checkArgument( - predecessor.getSelects().getStratificationDate().isPresent(), - "Expecting %s to contain a stratification date representing the min and max stratification bounds" + predecessor.getSelects().getStratificationDate().isPresent(), + "Expecting %s to contain a stratification date representing the min and max stratification bounds" ); ColumnDateRange finalRange = functionProvider.intersection( - stratificationFunctions.ofStartAndEnd(rangeStart, rangeEnd), - predecessor.getQualifiedSelects().getStratificationDate().get() - ) - .as(SharedAliases.STRATIFICATION_BOUNDS.getAlias()); + stratificationFunctions.ofStartAndEnd(rangeStart, rangeEnd), + predecessor.getQualifiedSelects().getStratificationDate().get() + ).as(SharedAliases.STRATIFICATION_BOUNDS.getAlias()); - Selects selects = Selects.builder() - .ids(ids) - .stratificationDate(Optional.of(finalRange)) - .build(); + Selects selects = Selects.builder().ids(ids).stratificationDate(Optional.of(finalRange)).build(); QueryStep.QueryStepBuilder queryStep = QueryStep.builder() - .selects(selects) - .fromTable(QueryStep.toTableLike(predecessor.getCteName())); + .selects(selects) + .fromTable( + QueryStep.toTableLike(predecessor.getCteName())); seriesTable.ifPresent(queryStep::fromTable); return queryStep.build(); @@ -259,8 +356,7 @@ private static Range toGenerateSeriesBounds(RelativeFormQuery relativeF if (resolution == Resolution.YEARS) { timeCountBefore = divideAndRoundUp(relativeForm.getTimeCountBefore(), 4); timeCountAfter = divideAndRoundUp(relativeForm.getTimeCountAfter(), 4); - } - else { + } else { timeCountBefore = relativeForm.getTimeCountBefore(); timeCountAfter = relativeForm.getTimeCountAfter(); } @@ -272,12 +368,20 @@ private static Range toGenerateSeriesBounds(RelativeFormQuery relativeF timeCountAfter = relativeForm.getTimeCountAfter(); } case YEARS -> { - timeCountBefore = divideAndRoundUp(relativeForm.getTimeCountBefore(), Interval.YEAR_AS_DAYS_INTERVAL.getAmount()); - timeCountAfter = divideAndRoundUp(relativeForm.getTimeCountAfter(), Interval.YEAR_AS_DAYS_INTERVAL.getAmount()); + timeCountBefore = divideAndRoundUp( + relativeForm.getTimeCountBefore(), + Interval.YEAR_AS_DAYS_INTERVAL.getAmount()); + timeCountAfter = divideAndRoundUp( + relativeForm.getTimeCountAfter(), + Interval.YEAR_AS_DAYS_INTERVAL.getAmount()); } case QUARTERS -> { - timeCountBefore = divideAndRoundUp(relativeForm.getTimeCountBefore(), Interval.NINETY_DAYS_INTERVAL.getAmount()); - timeCountAfter = divideAndRoundUp(relativeForm.getTimeCountAfter(), Interval.NINETY_DAYS_INTERVAL.getAmount()); + timeCountBefore = divideAndRoundUp( + relativeForm.getTimeCountBefore(), + Interval.NINETY_DAYS_INTERVAL.getAmount()); + timeCountAfter = divideAndRoundUp( + relativeForm.getTimeCountAfter(), + Interval.NINETY_DAYS_INTERVAL.getAmount()); } default -> throw new CombinationNotSupportedException(relativeForm.getTimeUnit(), resolution); } @@ -286,8 +390,8 @@ private static Range toGenerateSeriesBounds(RelativeFormQuery relativeF } return Range.of( - -timeCountBefore, - timeCountAfter + -timeCountBefore, + timeCountAfter ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java index 1a2478b4f1c..b9912b667b9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java @@ -124,7 +124,11 @@ public ColumnDateRange ofStartAndEnd(Field start, Field end) { /** * Shift's a start date by an interval times an amount. The offset will we added to the amount before multiplying. */ - public abstract Field shiftByInterval(Field startDate, Interval interval, Field amount, Offset offset); + public abstract Field shiftByInterval( + Field startDate, + Interval interval, + Field amount, + Offset offset); /** * Generates the start and end field for the respective {@link IndexPlacement} and {@link CalendarUnit timeUnit}. @@ -171,8 +175,8 @@ public List> indexStartFields(IndexPlacement indexPlacement, Calenda } return List.of( - positiveStart.as(SharedAliases.INDEX_START_POSITIVE.getAlias()), - negativeStart.as(SharedAliases.INDEX_START_NEGATIVE.getAlias()) + positiveStart.as(SharedAliases.INDEX_START_POSITIVE.getAlias()), + negativeStart.as(SharedAliases.INDEX_START_NEGATIVE.getAlias()) ); } @@ -182,21 +186,28 @@ public List> indexStartFields(IndexPlacement indexPlacement, Calenda * @return A {@link Field} representing the count of resolution windows. * @throws CombinationNotSupportedException if the combination of resolution and alignment is not supported. */ - public Field calculateResolutionWindowCount(ExportForm.ResolutionAndAlignment resolutionAndAlignment, ColumnDateRange bounds) { + public Field calculateResolutionWindowCount( + ExportForm.ResolutionAndAlignment resolutionAndAlignment, + ColumnDateRange bounds) { SqlFunctionProvider functionProvider = getFunctionProvider(); return switch (resolutionAndAlignment.getResolution()) { case COMPLETE -> DSL.inline(1); case YEARS -> calculateResolutionWindowForYearResolution(resolutionAndAlignment, bounds, functionProvider); - case QUARTERS -> calculateResolutionWindowForQuarterResolution(resolutionAndAlignment, bounds, functionProvider); - case DAYS -> functionProvider.dateDistance(ChronoUnit.DAYS, lower(bounds), exclusiveUpper(bounds)) - .as(SharedAliases.DAY_ALIGNED_COUNT.getAlias()); + case QUARTERS -> + calculateResolutionWindowForQuarterResolution(resolutionAndAlignment, bounds, functionProvider); + case DAYS -> + functionProvider.dateDistance(ChronoUnit.DAYS, lower(bounds), exclusiveUpper(bounds)) + .as( + SharedAliases.DAY_ALIGNED_COUNT.getAlias()); }; } /** * Determines the stratification range based on resolution and alignment parameters. The created stratification range is bound by the given range. */ - public ColumnDateRange createStratificationRange(ExportForm.ResolutionAndAlignment resolutionAndAlignment, ColumnDateRange bounds) { + public ColumnDateRange createStratificationRange( + ExportForm.ResolutionAndAlignment resolutionAndAlignment, + ColumnDateRange bounds) { SqlFunctionProvider functionProvider = getFunctionProvider(); @@ -215,16 +226,12 @@ public ColumnDateRange createStratificationRange(ExportForm.ResolutionAndAlignme */ public Field index(SqlIdColumns ids, Optional stratificationBounds) { - List> partitioningFields = - Stream.concat( - ids.toFields().stream(), - stratificationBounds.stream().flatMap(columnDateRange -> columnDateRange.toFields().stream()) - ) - .collect(Collectors.toList()); + List> partitioningFields = Stream.concat( + ids.toFields().stream(), + stratificationBounds.stream().flatMap(columnDateRange -> columnDateRange.toFields().stream()) + ).collect(Collectors.toList()); - return DSL.rowNumber() - .over(DSL.partitionBy(partitioningFields)) - .as(SharedAliases.INDEX.getAlias()); + return DSL.rowNumber().over(DSL.partitionBy(partitioningFields)).as(SharedAliases.INDEX.getAlias()); } /** @@ -242,41 +249,52 @@ public Condition stopOnMaxResolutionWindowCount(ExportForm.ResolutionAndAlignmen } private Field calculateResolutionWindowForQuarterResolution( - ExportForm.ResolutionAndAlignment resolutionAndAlignment, - ColumnDateRange bounds, - SqlFunctionProvider functionProvider + ExportForm.ResolutionAndAlignment resolutionAndAlignment, + ColumnDateRange bounds, + SqlFunctionProvider functionProvider ) { return switch (resolutionAndAlignment.getAlignment()) { - case QUARTER -> functionProvider.dateDistance(ChronoUnit.MONTHS, QUARTER_START, QUARTER_END) - .divide(MONTHS_PER_QUARTER) - .as(SharedAliases.QUARTER_ALIGNED_COUNT.getAlias()); - case DAY -> functionProvider.dateDistance(ChronoUnit.DAYS, lower(bounds), exclusiveUpper(bounds)) - .plus(89) - .divide(DAYS_PER_QUARTER) - .as(SharedAliases.DAY_ALIGNED_COUNT.getAlias()); + case QUARTER -> + functionProvider.dateDistance(ChronoUnit.MONTHS, QUARTER_START, QUARTER_END) + .divide( + MONTHS_PER_QUARTER) + .as(SharedAliases.QUARTER_ALIGNED_COUNT.getAlias()); + case DAY -> + functionProvider.dateDistance(ChronoUnit.DAYS, lower(bounds), exclusiveUpper(bounds)) + .plus(89) + .divide( + DAYS_PER_QUARTER) + .as(SharedAliases.DAY_ALIGNED_COUNT.getAlias()); default -> throw new CombinationNotSupportedException(resolutionAndAlignment); }; } private Field calculateResolutionWindowForYearResolution( - ExportForm.ResolutionAndAlignment resolutionAndAlignment, - ColumnDateRange bounds, - SqlFunctionProvider functionProvider + ExportForm.ResolutionAndAlignment resolutionAndAlignment, + ColumnDateRange bounds, + SqlFunctionProvider functionProvider ) { return switch (resolutionAndAlignment.getAlignment()) { - case YEAR -> functionProvider.dateDistance(ChronoUnit.YEARS, YEAR_START, YEAR_END) - .as(SharedAliases.YEAR_ALIGNED_COUNT.getAlias()); - case QUARTER -> functionProvider.dateDistance(ChronoUnit.YEARS, QUARTER_START, YEAR_END_QUARTER_ALIGNED) - .as(SharedAliases.QUARTER_ALIGNED_COUNT.getAlias()); - case DAY -> functionProvider.dateDistance(ChronoUnit.DAYS, lower(bounds), exclusiveUpper(bounds)) - .plus(364) - .divide(DAYS_PER_YEAR) - .as(SharedAliases.DAY_ALIGNED_COUNT.getAlias()); + case YEAR -> + functionProvider.dateDistance(ChronoUnit.YEARS, YEAR_START, YEAR_END) + .as( + SharedAliases.YEAR_ALIGNED_COUNT.getAlias()); + case QUARTER -> + functionProvider.dateDistance(ChronoUnit.YEARS, QUARTER_START, YEAR_END_QUARTER_ALIGNED) + .as( + SharedAliases.QUARTER_ALIGNED_COUNT.getAlias()); + case DAY -> + functionProvider.dateDistance(ChronoUnit.DAYS, lower(bounds), exclusiveUpper(bounds)) + .plus(364) + .divide( + DAYS_PER_YEAR) + .as(SharedAliases.DAY_ALIGNED_COUNT.getAlias()); default -> throw new CombinationNotSupportedException(resolutionAndAlignment); }; } - private ColumnDateRange createStratificationRangeForQuarterResolution(ExportForm.ResolutionAndAlignment resolutionAndAlignment) { + private ColumnDateRange createStratificationRangeForQuarterResolution( + ExportForm.ResolutionAndAlignment resolutionAndAlignment) { return switch (resolutionAndAlignment.getAlignment()) { case QUARTER -> calcRange(QUARTER_START, Interval.QUARTER_INTERVAL); case DAY -> calcRange(INDEX_START, Interval.NINETY_DAYS_INTERVAL); @@ -284,7 +302,8 @@ private ColumnDateRange createStratificationRangeForQuarterResolution(ExportForm }; } - private ColumnDateRange createStratificationRangeForYearResolution(ExportForm.ResolutionAndAlignment resolutionAndAlignment) { + private ColumnDateRange createStratificationRangeForYearResolution( + ExportForm.ResolutionAndAlignment resolutionAndAlignment) { return switch (resolutionAndAlignment.getAlignment()) { case YEAR -> calcRange(YEAR_START, Interval.ONE_YEAR_INTERVAL); case QUARTER -> calcRange(QUARTER_START, Interval.ONE_YEAR_INTERVAL); @@ -293,7 +312,9 @@ private ColumnDateRange createStratificationRangeForYearResolution(ExportForm.Re }; } - private static Condition windowCountForQuarterResolution(ExportForm.ResolutionAndAlignment resolutionAndAlignment, Field seriesIndex) { + private static Condition windowCountForQuarterResolution( + ExportForm.ResolutionAndAlignment resolutionAndAlignment, + Field seriesIndex) { return switch (resolutionAndAlignment.getAlignment()) { case QUARTER -> seriesIndex.lessOrEqual(QUARTER_ALIGNED_COUNT); case DAY -> seriesIndex.lessOrEqual(DAY_ALIGNED_COUNT); @@ -301,7 +322,9 @@ private static Condition windowCountForQuarterResolution(ExportForm.ResolutionAn }; } - private static Condition windowCountForYearResolution(ExportForm.ResolutionAndAlignment resolutionAndAlignment, Field seriesIndex) { + private static Condition windowCountForYearResolution( + ExportForm.ResolutionAndAlignment resolutionAndAlignment, + Field seriesIndex) { return switch (resolutionAndAlignment.getAlignment()) { case YEAR -> seriesIndex.lessOrEqual(YEAR_ALIGNED_COUNT); case QUARTER -> seriesIndex.lessOrEqual(QUARTER_ALIGNED_COUNT); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java index 4287a5ace82..562b109119e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java @@ -31,31 +31,42 @@ public StratificationTableFactory(QueryStep baseStep, ConversionContext context) } public QueryStep createRelativeStratificationTable(RelativeFormQuery form, ConversionContext context) { - RelativeStratification relativeStratification = new RelativeStratification(baseStep, stratificationFunctions, functionProvider); + RelativeStratification relativeStratification = new RelativeStratification( + baseStep, + stratificationFunctions, + functionProvider); return relativeStratification.createRelativeStratificationTable(form, context); } - public QueryStep createAbsoluteStratificationTable(List resolutionAndAlignments, ConversionContext context) { + public QueryStep createAbsoluteStratificationTable( + List resolutionAndAlignments, + ConversionContext context) { AbsoluteStratification absoluteStratification = new AbsoluteStratification(baseStep, stratificationFunctions); return absoluteStratification.createStratificationTable(resolutionAndAlignments, context); } - protected static QueryStep unionResolutionTables(List unionSteps, List predecessors, ConversionContext context) { + protected static QueryStep unionResolutionTables( + List unionSteps, + List predecessors, + ConversionContext context) { Preconditions.checkArgument(!unionSteps.isEmpty(), "Expecting at least 1 resolution table"); List withQualifiedSelects = unionSteps.stream() - .map(queryStep -> QueryStep.builder() - .selects(queryStep.getQualifiedSelects()) - .fromTable(QueryStep.toTableLike(queryStep.getCteName())) - .build()) - .toList(); + .map( + queryStep -> QueryStep.builder() + .selects(queryStep.getQualifiedSelects()) + .fromTable( + QueryStep.toTableLike(queryStep.getCteName())) + .build()) + .toList(); return QueryStep.createUnionAllStep( - withQualifiedSelects, - FormCteStep.FULL_STRATIFICATION.getSuffix(), - Stream.concat(predecessors.stream(), unionSteps.stream()).toList(), - context.isNegation(), context.getFunctionProvider() + withQualifiedSelects, + FormCteStep.FULL_STRATIFICATION.getSuffix(), + Stream.concat(predecessors.stream(), unionSteps.stream()).toList(), + context.isNegation(), + context.getFunctionProvider() ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java index f8679840ce8..7a7e1972f87 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java @@ -12,7 +12,7 @@ import org.jooq.Field; import org.jooq.impl.DSL; -//TODO split this class up into Dialect specific versions. +// TODO split this class up into Dialect specific versions. @Getter @ToString(onlyExplicitlyIncluded = true) public class ColumnDateRange implements SqlSelect { @@ -48,39 +48,35 @@ public ColumnDateRange asValidityDateRange(String alias) { @Override public List> toFields() { - return Stream.of(start, end) - .collect(Collectors.toList()); + return Stream.of(start, end).collect(Collectors.toList()); } @Override public ColumnDateRange qualify(String qualifier) { return new ColumnDateRange( - QualifyingUtil.qualify(getStart(), qualifier), - QualifyingUtil.qualify(getEnd(), qualifier), - getAlias() + QualifyingUtil.qualify(getStart(), qualifier), + QualifyingUtil.qualify(getEnd(), qualifier), + getAlias() ); } @Override public List requiredColumns() { - return toFields().stream() - .map(Field::getName) - .distinct() - .toList(); + return toFields().stream().map(Field::getName).distinct().toList(); } public ColumnDateRange as(String alias) { return new ColumnDateRange( - start.as(alias + START_SUFFIX), - end.as(alias + END_SUFFIX), - alias + start.as(alias + START_SUFFIX), + end.as(alias + END_SUFFIX), + alias ); } public ColumnDateRange coalesce(ColumnDateRange right) { return ColumnDateRange.of( - DSL.coalesce(start, right.getStart()), - DSL.coalesce(end, right.getEnd()) + DSL.coalesce(start, right.getStart()), + DSL.coalesce(end, right.getEnd()) ).as(alias); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ConqueryJoinType.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ConqueryJoinType.java index 3dee591ad95..fc9e9e5ffcb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ConqueryJoinType.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ConqueryJoinType.java @@ -4,7 +4,5 @@ * A subset of {@link org.jooq.JoinType} for join types that Conquery supports. */ public enum ConqueryJoinType { - INNER_JOIN, - OUTER_JOIN, - LEFT_JOIN + INNER_JOIN, OUTER_JOIN, LEFT_JOIN } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/CteStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/CteStep.java index 5f6833df01a..6e5ebc8511b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/CteStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/CteStep.java @@ -20,21 +20,26 @@ public interface CteStep { * but values might be null. */ static Map getDefaultPredecessorMap(Set requiredSteps) { - return requiredSteps.stream().collect( + return requiredSteps.stream() + .collect( HashMap::new, (map, cteStep) -> map.put(cteStep, cteStep.getPredecessor()), // value might be null Map::putAll - ); + ); } /** * Generates a CTE name for each of the given required steps. Combines the given label with the CTE step suffix (@link CteStep#getSuffix()). */ - static Map createCteNameMap(Set requiredSteps, String label, NameGenerator nameGenerator) { - return requiredSteps.stream().collect( + static Map createCteNameMap( + Set requiredSteps, + String label, + NameGenerator nameGenerator) { + return requiredSteps.stream() + .collect( Collectors.toMap( - Function.identity(), - step -> nameGenerator.cteStepName(step, label) + Function.identity(), + step -> nameGenerator.cteStepName(step, label) )); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java index 20004a4ff20..b3b75f7b5d0 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java @@ -69,7 +69,8 @@ public String joinedNodeName(ConqueryJoinType logicalOperation) { return switch (logicalOperation) { case INNER_JOIN -> "AND-%d".formatted(++andCount); case OUTER_JOIN -> "OR-%d".formatted(++orCount); - case LEFT_JOIN -> throw new UnsupportedOperationException("Creating CTE names for LEFT_JOIN nodes is not supported"); + case LEFT_JOIN -> + throw new UnsupportedOperationException("Creating CTE names for LEFT_JOIN nodes is not supported"); }; } @@ -83,8 +84,7 @@ private String ensureValidLength(String input) { } private static String lowerAndReplaceWhitespace(String name) { - return WHITESPACE.matcher(name.toLowerCase()) - .replaceAll("_"); + return WHITESPACE.matcher(name.toLowerCase()).replaceAll("_"); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NumberMapUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NumberMapUtil.java index f3a4f2362e2..7f48ef14ca5 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NumberMapUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NumberMapUtil.java @@ -9,10 +9,14 @@ public class NumberMapUtil { public static final Map> NUMBER_MAP = Map.of( - MajorTypeId.MONEY, BigDecimal.class, - MajorTypeId.DECIMAL, Double.class, - MajorTypeId.REAL, Double.class, - MajorTypeId.INTEGER, Integer.class + MajorTypeId.MONEY, + BigDecimal.class, + MajorTypeId.DECIMAL, + Double.class, + MajorTypeId.REAL, + Double.class, + MajorTypeId.INTEGER, + Integer.class ); public static Class getType(Column column) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QualifyingUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QualifyingUtil.java index a4665c49570..fe18b3acfba 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QualifyingUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QualifyingUtil.java @@ -14,9 +14,7 @@ public static Field qualify(Field field, String qualifier) { } public static List qualify(List sqlSelects, String qualifier) { - return sqlSelects.stream() - .map(sqlSelect -> sqlSelect.qualify(qualifier)) - .collect(Collectors.toList()); + return sqlSelects.stream().map(sqlSelect -> sqlSelect.qualify(qualifier)).collect(Collectors.toList()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java index ee3d96bcde2..26a4945bbae 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java @@ -1,5 +1,8 @@ package com.bakdata.conquery.sql.conversion.model; +import java.util.Collections; +import java.util.List; + import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; import lombok.Builder; @@ -11,9 +14,6 @@ import org.jooq.TableLike; import org.jooq.impl.DSL; -import java.util.Collections; -import java.util.List; - /** * Intermediate representation of an SQL query. */ @@ -62,23 +62,40 @@ public class QueryStep { @Singular List predecessors; - public static QueryStep createUnionAllStep(List unionSteps, String cteName, List predecessors, boolean negation, SqlFunctionProvider functionProvider) { + public static QueryStep createUnionAllStep( + List unionSteps, + String cteName, + List predecessors, + boolean negation, + SqlFunctionProvider functionProvider) { return createUnionStep(unionSteps, cteName, predecessors, true, negation, functionProvider); } - public static QueryStep createUnionStep(List unionSteps, String cteName, List predecessors, boolean negation, SqlFunctionProvider functionProvider) { + public static QueryStep createUnionStep( + List unionSteps, + String cteName, + List predecessors, + boolean negation, + SqlFunctionProvider functionProvider) { return createUnionStep(unionSteps, cteName, predecessors, false, negation, functionProvider); } - private static QueryStep createUnionStep(List unionSteps, String cteName, List predecessors, boolean unionAll, boolean negation, SqlFunctionProvider functionProvider) { + private static QueryStep createUnionStep( + List unionSteps, + String cteName, + List predecessors, + boolean unionAll, + boolean negation, + SqlFunctionProvider functionProvider) { QueryStep first = unionSteps.getFirst(); return first.toBuilder() - .cteName(cteName) - .union(unionSteps.stream().skip(1).toList()) - .unionAll(unionAll) - .predecessors(predecessors) - .negate(negation) - .build(); + .cteName(cteName) + .union(unionSteps.stream().skip(1).toList()) + .unionAll( + unionAll) + .predecessors(predecessors) + .negate(negation) + .build(); } public static TableLike toTableLike(String fromTableName) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java index 79f1e6532e5..d5ddeb55e72 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java @@ -36,7 +36,10 @@ public class QueryStepJoiner { /** * Implements an antijoin against the allIdsTable defined in {@link IdColumnConfig#getTable()}, for negation without siblings. */ - public static QueryStep antiJoinWithAllIdsTable(QueryStep queryStep, ConversionContext context, DateAggregationAction dateAggregationAction) { + public static QueryStep antiJoinWithAllIdsTable( + QueryStep queryStep, + ConversionContext context, + DateAggregationAction dateAggregationAction) { SqlFunctionProvider functionProvider = context.getConversionContext().getDialectBundle().getFunctionProvider(); @@ -47,9 +50,9 @@ public static QueryStep antiJoinWithAllIdsTable(QueryStep queryStep, ConversionC Field allIdsPrimaryColumn = field(name(allIdsTable, idColumnConfig.getField()), String.class); - Table table = table(name(allIdsTable)) - .leftOuterJoin(table(name(queryStep.getCteName()))) - .on(allIdsPrimaryColumn.eq(queryStepPrimaryColumn)); + Table table = table(name(allIdsTable)).leftOuterJoin(table(name(queryStep.getCteName()))) + .on( + allIdsPrimaryColumn.eq(queryStepPrimaryColumn)); String cteName = queryStep.getCteName() + NEGATED_CTE_SUFFIX; @@ -59,24 +62,26 @@ public static QueryStep antiJoinWithAllIdsTable(QueryStep queryStep, ConversionC }; Selects selects = Selects.builder() - .ids(new SqlIdColumns(allIdsPrimaryColumn)) - .validityDate(validityDate.map(cdr -> cdr.asValidityDateRange(cteName))) - .build(); + .ids(new SqlIdColumns(allIdsPrimaryColumn)) + .validityDate( + validityDate.map(cdr -> cdr.asValidityDateRange(cteName))) + .build(); return QueryStep.builder() - .cteName(cteName) - .selects(selects) - .fromTable(table) - .conditions(List.of(queryStepPrimaryColumn.isNull())) - .predecessor(queryStep) - .build(); + .cteName(cteName) + .selects(selects) + .fromTable(table) + .conditions( + List.of(queryStepPrimaryColumn.isNull())) + .predecessor(queryStep) + .build(); } public static QueryStep joinChildren( - Iterable children, - ConversionContext context, - ConqueryJoinType logicalOperation, - DateAggregationAction dateAggregationAction + Iterable children, + ConversionContext context, + ConqueryJoinType logicalOperation, + DateAggregationAction dateAggregationAction ) { ConversionContext childrenContext = context.createChildContext(); @@ -89,10 +94,10 @@ public static QueryStep joinChildren( } public static QueryStep joinSteps( - List queriesToJoin, - ConqueryJoinType logicalOperation, - DateAggregationAction dateAggregationAction, - ConversionContext context + List queriesToJoin, + ConqueryJoinType logicalOperation, + DateAggregationAction dateAggregationAction, + ConversionContext context ) { //TODO somehow Negation does not show up here. @@ -106,10 +111,10 @@ public static QueryStep joinSteps( } private static QueryStep doJoin( - List queriesToJoin, - ConqueryJoinType logicalOperation, - DateAggregationAction dateAggregationAction, - ConversionContext context) { + List queriesToJoin, + ConqueryJoinType logicalOperation, + DateAggregationAction dateAggregationAction, + ConversionContext context) { // no join required if (queriesToJoin.size() == 1) { return queriesToJoin.getFirst(); @@ -122,9 +127,10 @@ private static QueryStep doJoin( QueryStep joinedStep; QueryStep.QueryStepBuilder joinedStepBuilder = QueryStep.builder() - .cteName(joinedNodeName) - .fromTable(joinedTable) - .predecessors(queriesToJoin); + .cteName(joinedNodeName) + .fromTable( + joinedTable) + .predecessors(queriesToJoin); DateAggregationDates dateAggregationDates = DateAggregationDates.forSteps(queriesToJoin); @@ -136,10 +142,20 @@ private static QueryStep doJoin( // if there is only 1 child node containing a validity date, we just keep it as overall validity date for the joined node else if (dateAggregationDates.getValidityDates().size() == 1) { ColumnDateRange validityDate = dateAggregationDates.getValidityDates().getFirst(); - joinedStep = buildJoinedStep(ids, mergedSelects, Optional.of(validityDate), Optional.empty(), joinedStepBuilder); - } - else { - joinedStep = buildStepAndAggregateDates(ids, mergedSelects, joinedStepBuilder, dateAggregationDates, dateAggregationAction, context); + joinedStep = buildJoinedStep( + ids, + mergedSelects, + Optional.of(validityDate), + Optional.empty(), + joinedStepBuilder); + } else { + joinedStep = buildStepAndAggregateDates( + ids, + mergedSelects, + joinedStepBuilder, + dateAggregationDates, + dateAggregationAction, + context); } return joinedStep; } @@ -150,12 +166,14 @@ else if (dateAggregationDates.getValidityDates().size() == 1) { * That means the negation works by removing the entities in the negated portion from the unnegated ones. */ private static QueryStep joinStepsContainingNegation( - List queriesToJoin, - ConqueryJoinType logicalOperation, - DateAggregationAction dateAggregationAction, - ConversionContext context + List queriesToJoin, + ConqueryJoinType logicalOperation, + DateAggregationAction dateAggregationAction, + ConversionContext context ) { - Map> byNegation = queriesToJoin.stream().collect(Collectors.groupingBy(QueryStep::isNegate)); + Map> byNegation = queriesToJoin.stream() + .collect( + Collectors.groupingBy(QueryStep::isNegate)); List withoutNegation = byNegation.get(false); List withNegation = byNegation.get(true); @@ -170,23 +188,30 @@ private static QueryStep joinStepsContainingNegation( QueryStep negateJoined = doJoin(withNegation, logicalOperation, dateAggregationAction, context); Selects.SelectsBuilder selects = nonNegateJoined.getQualifiedSelects() - .toBuilder() - .sqlSelects(mergeSelects(List.of(nonNegateJoined, negateJoined))); + .toBuilder() + .sqlSelects( + mergeSelects(List.of(nonNegateJoined, negateJoined))); if (logicalOperation == ConqueryJoinType.INNER_JOIN) { - Table table = table(name(nonNegateJoined.getCteName())) - .leftOuterJoin(table(name(negateJoined.getCteName()))) - .on(nonNegateJoined.getQualifiedSelects().getIds().getPrimaryColumn() - .eq(negateJoined.getQualifiedSelects().getIds().getPrimaryColumn())); + Table table = table(name(nonNegateJoined.getCteName())).leftOuterJoin( + table(name(negateJoined.getCteName()))) + .on( + nonNegateJoined.getQualifiedSelects() + .getIds() + .getPrimaryColumn() + .eq( + negateJoined.getQualifiedSelects().getIds().getPrimaryColumn())); return QueryStep.builder() - .cteName(cteName) - .selects(selects.build()) - .fromTable(table) - .conditions(List.of(negateJoined.getQualifiedSelects().getIds().getPrimaryColumn().isNull())) - .predecessors(List.of(nonNegateJoined, negateJoined)) - .build(); + .cteName(cteName) + .selects(selects.build()) + .fromTable(table) + .conditions( + List.of(negateJoined.getQualifiedSelects().getIds().getPrimaryColumn().isNull())) + .predecessors( + List.of(nonNegateJoined, negateJoined)) + .build(); } // first, invert dates of negated step @@ -201,48 +226,53 @@ private static QueryStep joinStepsContainingNegation( // prepare date aggregation Condition infinityRangeCondition = negatePrimaryColumn.isNull().or(nonNegatePrimaryColumn.isNull()); - DateAggregationDates aggregationDates = DateAggregationDates.forValidityDates(List.of( + DateAggregationDates aggregationDates = DateAggregationDates.forValidityDates( + List.of( nonNegateJoined.getQualifiedSelects().getValidityDate(), negateJoined.getQualifiedSelects().getValidityDate(), Optional.of(context.getDialectBundle().getFunctionProvider().allRangeIf(infinityRangeCondition)) - )); - ColumnDateRange merged = - dateAggregator.getAggregatedValidityDate(aggregationDates, DateAggregationAction.MERGE) - .as(cteName + SharedAliases.DATES_COLUMN.getAlias()); + )); + ColumnDateRange merged = dateAggregator.getAggregatedValidityDate( + aggregationDates, + DateAggregationAction.MERGE).as(cteName + SharedAliases.DATES_COLUMN.getAlias()); Field coalescedId = DSL.coalesce(nonNegatePrimaryColumn, allIdsPrimaryColumn) - .as(SharedAliases.PRIMARY_COLUMN.getAlias()); + .as( + SharedAliases.PRIMARY_COLUMN.getAlias()); - selects = selects - .ids(new SqlIdColumns(coalescedId)) - .validityDate(Optional.of(merged)); + selects = selects.ids(new SqlIdColumns(coalescedId)).validityDate(Optional.of(merged)); - Table table = table(name(context.getIdColumns().getTable())) - .leftOuterJoin(table(name(negateJoined.getCteName()))) - .on(allIdsPrimaryColumn.eq(negatePrimaryColumn)) - .leftOuterJoin(table(name(nonNegateJoined.getCteName()))) - .on(allIdsPrimaryColumn.eq(nonNegatePrimaryColumn)); + Table table = table(name(context.getIdColumns().getTable())).leftOuterJoin( + table(name(negateJoined.getCteName()))) + .on(allIdsPrimaryColumn.eq(negatePrimaryColumn)) + .leftOuterJoin( + table(name(nonNegateJoined.getCteName()))) + .on(allIdsPrimaryColumn.eq(nonNegatePrimaryColumn)); return QueryStep.builder() - .cteName(cteName) - .selects(selects.build()) - .fromTable(table) - .conditions(List.of(negatePrimaryColumn.isNull().or(nonNegatePrimaryColumn.isNotNull()))) - .predecessors(List.of(nonNegateJoined, negateJoined)) - .build(); + .cteName(cteName) + .selects(selects.build()) + .fromTable(table) + .conditions( + List.of(negatePrimaryColumn.isNull().or(nonNegatePrimaryColumn.isNotNull()))) + .predecessors( + List.of(nonNegateJoined, negateJoined)) + .build(); } - public static TableLike constructJoinedTable(List queriesToJoin, ConqueryJoinType logicalOperation, ConversionContext context) { + public static TableLike constructJoinedTable( + List queriesToJoin, + ConqueryJoinType logicalOperation, + ConversionContext context) { SqlFunctionProvider functionProvider = context.getFunctionProvider(); - Function3, Table, List, TableOnConditionStep> joinType = - switch (logicalOperation) { - case INNER_JOIN -> functionProvider::innerJoin; - case OUTER_JOIN -> functionProvider::fullOuterJoin; - case LEFT_JOIN -> functionProvider::leftJoin; - }; + Function3, Table, List, TableOnConditionStep> joinType = switch (logicalOperation) { + case INNER_JOIN -> functionProvider::innerJoin; + case OUTER_JOIN -> functionProvider::fullOuterJoin; + case LEFT_JOIN -> functionProvider::leftJoin; + }; Table joinedQuery = getIntitialJoinTable(queriesToJoin); @@ -258,7 +288,9 @@ public static TableLike constructJoinedTable(List queriesToJo Condition joinDateCondition = joinOnStratification(leftPartQS, rightPartQS); - List joinConditions = Stream.concat(joinIdsCondition.stream(), Stream.of(joinDateCondition)).collect(Collectors.toList()); + List joinConditions = Stream.concat( + joinIdsCondition.stream(), + Stream.of(joinDateCondition)).collect(Collectors.toList()); Table rightPartTable = table(name(rightPartQS.getCteName())); joinedQuery = joinType.apply(joinedQuery, rightPartTable, joinConditions); @@ -271,7 +303,9 @@ public static TableLike constructJoinedTable(List queriesToJo * join on stratification date if present */ private static Condition joinOnStratification(QueryStep leftPartQS, QueryStep rightPartQS) { - if (leftPartQS.getSelects().getStratificationDate().isEmpty() || rightPartQS.getSelects().getStratificationDate().isEmpty()) { + if (leftPartQS.getSelects().getStratificationDate().isEmpty() || rightPartQS.getSelects() + .getStratificationDate() + .isEmpty()) { //TODO use unconditionalJoin in the future. Hana does not like noCondition joins return DSL.noCondition(); } @@ -284,8 +318,9 @@ private static Condition joinOnStratification(QueryStep leftPartQS, QueryStep ri public static List mergeSelects(List querySteps) { return querySteps.stream() - .flatMap(queryStep -> queryStep.getQualifiedSelects().getSqlSelects().stream()) - .collect(Collectors.toList()); + .flatMap( + queryStep -> queryStep.getQualifiedSelects().getSqlSelects().stream()) + .collect(Collectors.toList()); } public static SqlIdColumns coalesceIds(List querySteps) { @@ -299,28 +334,29 @@ private static Table getIntitialJoinTable(List queriesToJoin) } private static QueryStep buildJoinedStep( - SqlIdColumns ids, - List mergedSelects, - Optional validityDate, - Optional stratificationDate, - QueryStep.QueryStepBuilder builder + SqlIdColumns ids, + List mergedSelects, + Optional validityDate, + Optional stratificationDate, + QueryStep.QueryStepBuilder builder ) { Selects selects = Selects.builder() - .ids(ids) - .stratificationDate(stratificationDate) - .validityDate(validityDate) - .sqlSelects(mergedSelects) - .build(); + .ids(ids) + .stratificationDate(stratificationDate) + .validityDate( + validityDate) + .sqlSelects(mergedSelects) + .build(); return builder.selects(selects).build(); } private static QueryStep buildStepAndAggregateDates( - SqlIdColumns ids, - List mergedSelects, - QueryStep.QueryStepBuilder builder, - DateAggregationDates dateAggregationDates, - DateAggregationAction dateAggregationAction, - ConversionContext context + SqlIdColumns ids, + List mergedSelects, + QueryStep.QueryStepBuilder builder, + DateAggregationDates dateAggregationDates, + DateAggregationAction dateAggregationAction, + ConversionContext context ) { List withAllValidityDates = new ArrayList<>(mergedSelects); withAllValidityDates.addAll(dateAggregationDates.allStartsAndEnds()); @@ -328,21 +364,22 @@ private static QueryStep buildStepAndAggregateDates( SqlDateAggregator sqlDateAggregator = context.getDialectBundle().getDateAggregator(); return sqlDateAggregator.apply( - joinedStep, - mergedSelects, - dateAggregationDates, - dateAggregationAction, - context + joinedStep, + mergedSelects, + dateAggregationDates, + dateAggregationAction, + context ); } private static Optional coalesceStratificationDates(List queriesToJoin) { return queriesToJoin.stream() - .map(QueryStep::getQualifiedSelects) - .map(Selects::getStratificationDate) - .filter(Optional::isPresent) - .map(Optional::get) - .reduce(ColumnDateRange::coalesce); + .map(QueryStep::getQualifiedSelects) + .map(Selects::getStratificationDate) + .filter( + Optional::isPresent) + .map(Optional::get) + .reduce(ColumnDateRange::coalesce); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java index fa34d101b0f..d26dd0c9bda 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java @@ -1,14 +1,14 @@ package com.bakdata.conquery.sql.conversion.model; +import java.util.List; +import java.util.stream.Stream; + import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import lombok.RequiredArgsConstructor; import org.jooq.*; import org.jooq.Record; import org.jooq.impl.DSL; -import java.util.List; -import java.util.stream.Stream; - /** * Transformer for translating the intermediate representation of {@link QueryStep} into the final SQL query. */ @@ -22,12 +22,17 @@ public class QueryStepTransformer { */ public Select toSelectQuery(QueryStep queryStep, SqlFunctionProvider functionProvider) { - List> finalRepresentation = queryStep.getSelects().toFinalRepresentation(functionProvider, queryStep.isForTableExport()); + List> finalRepresentation = queryStep.getSelects() + .toFinalRepresentation( + functionProvider, + queryStep.isForTableExport()); - SelectConditionStep queryBase = this.dslContext.with(constructPredecessorCteList(queryStep, functionProvider)) - .select(finalRepresentation) - .from(queryStep.getFromTables()) - .where(queryStep.getConditions()); + SelectConditionStep queryBase = this.dslContext.with( + constructPredecessorCteList(queryStep, functionProvider)) + .select(finalRepresentation) + .from( + queryStep.getFromTables()) + .where(queryStep.getConditions()); // grouping SelectHavingStep grouped = queryBase; @@ -43,21 +48,26 @@ public Select toSelectQuery(QueryStep queryStep, SqlFunctionProvider fun return grouped; } - private List> constructPredecessorCteList(QueryStep queryStep, SqlFunctionProvider functionProvider) { - return predecessorCtes(queryStep, functionProvider) - .toList(); + private List> constructPredecessorCteList( + QueryStep queryStep, + SqlFunctionProvider functionProvider) { + return predecessorCtes(queryStep, functionProvider).toList(); } private List> toCteList(QueryStep queryStep, SqlFunctionProvider functionProvider) { return Stream.concat( - this.predecessorCtes(queryStep, functionProvider), - Stream.of(toCte(queryStep, functionProvider)) + this.predecessorCtes(queryStep, functionProvider), + Stream.of(toCte(queryStep, functionProvider)) ).toList(); } - private Stream> predecessorCtes(QueryStep queryStep, SqlFunctionProvider functionProvider) { - return queryStep.getPredecessors().stream() - .flatMap(predecessor -> toCteList(predecessor, functionProvider).stream()); + private Stream> predecessorCtes( + QueryStep queryStep, + SqlFunctionProvider functionProvider) { + return queryStep.getPredecessors() + .stream() + .flatMap( + predecessor -> toCteList(predecessor, functionProvider).stream()); } private CommonTableExpression toCte(QueryStep queryStep, SqlFunctionProvider functionProvider) { @@ -69,7 +79,10 @@ private Select toSelectStep(QueryStep queryStep, SqlFunctionProvider fun SelectSelectStep selectClause; - List> allSelects = queryStep.isForTableExport() ? queryStep.getSelects().toFinalRepresentation(functionProvider, true) : queryStep.getSelects().all(); + List> allSelects = queryStep.isForTableExport() ? queryStep.getSelects() + .toFinalRepresentation( + functionProvider, + true) : queryStep.getSelects().all(); if (queryStep.isSelectDistinct()) { selectClause = dslContext.selectDistinct(allSelects); @@ -92,11 +105,9 @@ private Select toSelectStep(QueryStep queryStep, SqlFunctionProvider fun private Select union(QueryStep queryStep, Select base, SqlFunctionProvider functionProvider) { for (QueryStep unionStep : queryStep.getUnion()) { - Select selectStep = - queryStep.isForTableExport() ? - // TODO this feels like a leaked abstraction, but i am not able to find the proper injection layer at the moment. - toSelectQuery(unionStep, functionProvider) : - toSelectStep(unionStep, functionProvider); + Select selectStep = queryStep.isForTableExport() ? + // TODO this feels like a leaked abstraction, but i am not able to find the proper injection layer at the moment. + toSelectQuery(unionStep, functionProvider) : toSelectStep(unionStep, functionProvider); if (queryStep.isUnionAll()) { base = base.unionAll(selectStep); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java index 160bae77115..ca771f66595 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java @@ -1,5 +1,11 @@ package com.bakdata.conquery.sql.conversion.model; +import java.util.List; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; @@ -9,12 +15,6 @@ import lombok.Value; import org.jooq.Field; -import java.util.List; -import java.util.Optional; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; - @Value @Builder(toBuilder = true) public class Selects { @@ -29,45 +29,43 @@ public class Selects { public List> toFinalRepresentation(SqlFunctionProvider functionProvider, boolean forTableExport) { - final Optional> validityDateRendered = - !forTableExport - ? getValidityDate().map(vdd -> functionProvider.dateRangeAggregation(vdd).as(SharedAliases.DATES_COLUMN.getAlias())) - : getValidityDate().map(vdd -> functionProvider.dateRangeToField(vdd).as(SharedAliases.DATES_COLUMN.getAlias())); + final Optional> validityDateRendered = !forTableExport ? getValidityDate().map( + vdd -> functionProvider.dateRangeAggregation(vdd) + .as( + SharedAliases.DATES_COLUMN.getAlias())) : getValidityDate().map( + vdd -> functionProvider.dateRangeToField(vdd).as(SharedAliases.DATES_COLUMN.getAlias())); - final Optional> stratificationDateRendered = getStratificationDate().map(vdd -> functionProvider.dateRangeToField(vdd).as(SharedAliases.STRATIFICATION_BOUNDS.getAlias())); + final Optional> stratificationDateRendered = getStratificationDate().map( + vdd -> functionProvider.dateRangeToField(vdd).as(SharedAliases.STRATIFICATION_BOUNDS.getAlias())); return Stream.of( - getIds().toFields().stream(), - stratificationDateRendered.stream(), - validityDateRendered.stream(), - getSqlSelects().stream().flatMap(sqlSelect -> sqlSelect.toFinalRepresentation().toFields().stream()) - ) - .flatMap(Function.identity()) - .map(select -> (Field) select) - .distinct() - .collect(Collectors.toList()); + getIds().toFields().stream(), + stratificationDateRendered.stream(), + validityDateRendered.stream(), + getSqlSelects().stream().flatMap(sqlSelect -> sqlSelect.toFinalRepresentation().toFields().stream()) + ).flatMap(Function.identity()).map(select -> (Field) select).distinct().collect(Collectors.toList()); } public Selects blockValidityDate() { - return this.toBuilder() - .validityDate(Optional.empty()) - .build(); + return this.toBuilder().validityDate(Optional.empty()).build(); } public Selects qualify(String qualifier) { SqlIdColumns ids = this.ids.qualify(qualifier); - List sqlSelects = this.sqlSelects.stream().map(sqlSelect -> sqlSelect.qualify(qualifier)).collect(Collectors.toList()); + List sqlSelects = this.sqlSelects.stream() + .map(sqlSelect -> sqlSelect.qualify(qualifier)) + .collect( + Collectors.toList()); - SelectsBuilder builder = Selects.builder() - .ids(ids) - .sqlSelects(sqlSelects); + SelectsBuilder builder = Selects.builder().ids(ids).sqlSelects(sqlSelects); if (this.validityDate.isPresent()) { builder = builder.validityDate(this.validityDate.map(_validityDate -> _validityDate.qualify(qualifier))); } if (this.stratificationDate.isPresent()) { - builder = builder.stratificationDate(this.stratificationDate.map(_validityDate -> _validityDate.qualify(qualifier))); + builder = builder.stratificationDate( + this.stratificationDate.map(_validityDate -> _validityDate.qualify(qualifier))); } return builder.build(); @@ -75,15 +73,11 @@ public Selects qualify(String qualifier) { public List> all() { return Stream.of( - this.ids.toFields().stream(), - this.stratificationDate.stream().flatMap(range -> range.toFields().stream()), - this.validityDate.stream().flatMap(range -> range.toFields().stream()), - this.sqlSelects.stream().flatMap(sqlSelect -> sqlSelect.toFields().stream()) - ) - .flatMap(Function.identity()) - .map(select -> (Field) select) - .distinct() - .collect(Collectors.toList()); + this.ids.toFields().stream(), + this.stratificationDate.stream().flatMap(range -> range.toFields().stream()), + this.validityDate.stream().flatMap(range -> range.toFields().stream()), + this.sqlSelects.stream().flatMap(sqlSelect -> sqlSelect.toFields().stream()) + ).flatMap(Function.identity()).map(select -> (Field) select).distinct().collect(Collectors.toList()); } /** @@ -91,14 +85,10 @@ public List> all() { */ public List> nonExplicitSelects() { return Stream.of( - this.ids.toFields().stream(), - this.stratificationDate.stream().flatMap(range -> range.toFields().stream()), - this.validityDate.stream().flatMap(range -> range.toFields().stream()) - ) - .flatMap(Function.identity()) - .map(select -> (Field) select) - .distinct() - .collect(Collectors.toList()); + this.ids.toFields().stream(), + this.stratificationDate.stream().flatMap(range -> range.toFields().stream()), + this.validityDate.stream().flatMap(range -> range.toFields().stream()) + ).flatMap(Function.identity()).map(select -> (Field) select).distinct().collect(Collectors.toList()); } /** @@ -106,9 +96,10 @@ public List> nonExplicitSelects() { */ public List> explicitSelects() { return this.sqlSelects.stream() - .flatMap(sqlSelect -> sqlSelect.toFields().stream()) - .distinct() - .collect(Collectors.toList()); + .flatMap(sqlSelect -> sqlSelect.toFields().stream()) + .distinct() + .collect( + Collectors.toList()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java index 99af726ccb3..0a567ce94ee 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java @@ -4,9 +4,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; - import javax.annotation.Nullable; import com.bakdata.conquery.models.forms.util.Resolution; @@ -51,9 +48,9 @@ public SqlIdColumns withAlias() { return new SqlIdColumns(this.primaryColumn.as(SharedAliases.PRIMARY_COLUMN.getAlias()), null, this); } return new SqlIdColumns( - this.primaryColumn.as(SharedAliases.PRIMARY_COLUMN.getAlias()), - this.secondaryId.as(SharedAliases.SECONDARY_ID.getAlias()), - this + this.primaryColumn.as(SharedAliases.PRIMARY_COLUMN.getAlias()), + this.secondaryId.as(SharedAliases.SECONDARY_ID.getAlias()), + this ); } @@ -70,23 +67,25 @@ public SqlIdColumns qualify(String qualifier) { public SqlIdColumns withAbsoluteStratification(Resolution resolution, Field index) { Field resolutionField = DSL.inline(resolution.toString()).as(SharedAliases.RESOLUTION.getAlias()); return StratificationSqlIdColumns.builder() - .primaryColumn(this.primaryColumn) - .secondaryId(this.secondaryId) - .resolution(resolutionField) - .index(index) - .eventDate(null) - .build(); + .primaryColumn(this.primaryColumn) + .secondaryId( + this.secondaryId) + .resolution(resolutionField) + .index(index) + .eventDate(null) + .build(); } public SqlIdColumns withRelativeStratification(Resolution resolution, Field index, Field eventDate) { Field resolutionField = DSL.inline(resolution.toString()).as(SharedAliases.RESOLUTION.getAlias()); return StratificationSqlIdColumns.builder() - .primaryColumn(this.primaryColumn) - .secondaryId(this.secondaryId) - .resolution(resolutionField) - .index(index) - .eventDate(eventDate) - .build(); + .primaryColumn(this.primaryColumn) + .secondaryId( + this.secondaryId) + .resolution(resolutionField) + .index(index) + .eventDate(eventDate) + .build(); } public SqlIdColumns forFinalSelect() { @@ -106,7 +105,7 @@ public boolean isWithStratification() { } public List> toFields() { - if (getSecondaryId().isEmpty()){ + if (getSecondaryId().isEmpty()) { return List.of(getPrimaryColumn()); } @@ -119,9 +118,8 @@ public List join(SqlIdColumns rightIds) { Condition joinPrimariesCondition = primaryColumn.eq(rightIds.getPrimaryColumn()); // join on secondary IDs if both are present - Condition joinSecondaries = getSecondaryId() - .flatMap(leftSecondaryId -> rightIds.getSecondaryId().map(leftSecondaryId::eq)) - .orElse(DSL.noCondition()); + Condition joinSecondaries = getSecondaryId().flatMap( + leftSecondaryId -> rightIds.getSecondaryId().map(leftSecondaryId::eq)).orElse(DSL.noCondition()); return List.of(joinPrimariesCondition, joinSecondaries); } @@ -141,11 +139,15 @@ public SqlIdColumns coalesce(List selectsIds) { ids.getSecondaryId().ifPresent(secondaryIds::add); }); - Field coalescedPrimaryColumn = coalesceFields(primaryColumns, String.class).coerce(String.class).as(SharedAliases.PRIMARY_COLUMN.getAlias()); + Field coalescedPrimaryColumn = coalesceFields(primaryColumns, String.class).coerce(String.class) + .as( + SharedAliases.PRIMARY_COLUMN.getAlias()); if (secondaryIds.isEmpty()) { return new SqlIdColumns(coalescedPrimaryColumn); } - Field coalescedSecondaryIds = coalesceFields(secondaryIds, String.class).coerce(String.class).as(SharedAliases.SECONDARY_ID.getAlias()); + Field coalescedSecondaryIds = coalesceFields(secondaryIds, String.class).coerce(String.class) + .as( + SharedAliases.SECONDARY_ID.getAlias()); return new SqlIdColumns(coalescedPrimaryColumn, coalescedSecondaryIds); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/StratificationSqlIdColumns.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/StratificationSqlIdColumns.java index 678afefab5c..01ff1ebae23 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/StratificationSqlIdColumns.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/StratificationSqlIdColumns.java @@ -6,7 +6,6 @@ import java.util.Objects; import java.util.stream.Collectors; import java.util.stream.Stream; - import javax.annotation.Nullable; import com.bakdata.conquery.models.forms.util.Resolution; @@ -44,12 +43,13 @@ public SqlIdColumns qualify(String qualifier) { } return StratificationSqlIdColumns.builder() - .primaryColumn(primaryColumn) - .secondaryId(null) - .resolution(resolution) - .index(index) - .eventDate(eventDate) - .build(); + .primaryColumn(primaryColumn) + .secondaryId(null) + .resolution( + resolution) + .index(index) + .eventDate(eventDate) + .build(); } /** @@ -59,19 +59,18 @@ public SqlIdColumns qualify(String qualifier) { public SqlIdColumns forFinalSelect() { Field withNulledCompleteIndex = DSL.when( - this.resolution.eq(DSL.inline(Resolution.COMPLETE.toString().toUpperCase())), - DSL.inline(null, Integer.class) - ) - .otherwise(this.index) - .as(SharedAliases.INDEX.getAlias()); + this.resolution.eq(DSL.inline(Resolution.COMPLETE.toString().toUpperCase())), + DSL.inline(null, Integer.class) + ).otherwise(this.index).as(SharedAliases.INDEX.getAlias()); return StratificationSqlIdColumns.builder() - .primaryColumn(getPrimaryColumn()) - .secondaryId(null) - .resolution(this.resolution) - .index(withNulledCompleteIndex) - .eventDate(this.eventDate) - .build(); + .primaryColumn(getPrimaryColumn()) + .secondaryId(null) + .resolution( + this.resolution) + .index(withNulledCompleteIndex) + .eventDate(this.eventDate) + .build(); } @Override @@ -82,13 +81,11 @@ public boolean isWithStratification() { @Override public List> toFields() { return Stream.of( - getPrimaryColumn(), - this.resolution, - this.index, - this.eventDate - ) - .filter(Objects::nonNull) - .collect(Collectors.toList()); + getPrimaryColumn(), + this.resolution, + this.index, + this.eventDate + ).filter(Objects::nonNull).collect(Collectors.toList()); } @Override @@ -99,29 +96,29 @@ public List join(SqlIdColumns rightIds) { } StratificationSqlIdColumns rightStratificationIds = (StratificationSqlIdColumns) rightIds; - Condition joinResolutionAndIndex = this.resolution.eq(rightStratificationIds.getResolution()).and(this.index.eq(rightStratificationIds.getIndex())); + Condition joinResolutionAndIndex = this.resolution.eq(rightStratificationIds.getResolution()) + .and( + this.index.eq(rightStratificationIds.getIndex())); Condition joinEventDateCondition; if (this.eventDate != null) { joinEventDateCondition = this.eventDate.eq(rightStratificationIds.getEventDate()); - } - else { + } else { joinEventDateCondition = DSL.noCondition(); } return Stream.concat( - super.join(rightIds).stream(), - Stream.of(joinResolutionAndIndex, joinEventDateCondition) - ) - .toList(); + super.join(rightIds).stream(), + Stream.of(joinResolutionAndIndex, joinEventDateCondition) + ).toList(); } @Override public SqlIdColumns coalesce(List selectsIds) { Preconditions.checkArgument( - selectsIds.stream().allMatch(SqlIdColumns::isWithStratification), - "Can only coalesce SqlIdColumns if all are with stratification" + selectsIds.stream().allMatch(SqlIdColumns::isWithStratification), + "Can only coalesce SqlIdColumns if all are with stratification" ); List> primaryColumns = new ArrayList<>(); @@ -147,8 +144,11 @@ public SqlIdColumns coalesce(List selectsIds) { } } - Field coalescedPrimaryColumn = coalesceFields(primaryColumns, String.class).coerce(String.class).as(SharedAliases.PRIMARY_COLUMN.getAlias()); - Field coalescedResolutions = coalesceFields(resolutions, String.class).as(SharedAliases.RESOLUTION.getAlias()); + Field coalescedPrimaryColumn = coalesceFields(primaryColumns, String.class).coerce(String.class) + .as( + SharedAliases.PRIMARY_COLUMN.getAlias()); + Field coalescedResolutions = coalesceFields(resolutions, String.class).as( + SharedAliases.RESOLUTION.getAlias()); Field coalescedIndices = coalesceFields(indices, Integer.class).as(SharedAliases.INDEX.getAlias()); Field eventDate = null; if (!eventDates.isEmpty()) { @@ -156,11 +156,12 @@ public SqlIdColumns coalesce(List selectsIds) { } return StratificationSqlIdColumns.builder() - .primaryColumn(coalescedPrimaryColumn) - .secondaryId(null) - .resolution(coalescedResolutions) - .index(coalescedIndices) - .eventDate(eventDate) - .build(); + .primaryColumn(coalescedPrimaryColumn) + .secondaryId(null) + .resolution( + coalescedResolutions) + .index(coalescedIndices) + .eventDate(eventDate) + .build(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java index e0ebf555de6..7d2eefb7e5d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java @@ -29,31 +29,40 @@ import org.jooq.Param; import org.jooq.impl.DSL; -//TODO(FK): this needs a rework. Current implementation makes a sum of the quarters and doesn't take overlapping events into account. +// TODO(FK): this needs a rework. Current implementation makes a sum of the quarters and doesn't take overlapping events +// into account. public class CountQuartersSqlAggregator implements SelectConverter, FilterConverter, SqlAggregator { private static CommonAggregationSelect createSingleDateColumnAggregationSelect( - Column countColumn, - String alias, - ConnectorSqlTables tables, - SqlFunctionProvider functionProvider) { + Column countColumn, + String alias, + ConnectorSqlTables tables, + SqlFunctionProvider functionProvider) { - ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(tables.getRootTable(), countColumn.getName(), Date.class); + ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>( + tables.getRootTable(), + countColumn.getName(), + Date.class); Field qualifiedRootSelect = rootSelect.qualify(tables.cteName(ConceptCteStep.PREPROCESSING)).select(); - FieldWrapper countQuartersAggregation = - new FieldWrapper<>(DSL.nullif(DSL.countDistinct(functionProvider.yearQuarter(qualifiedRootSelect)), 0).as(alias), countColumn.getName()); - - return CommonAggregationSelect.builder().rootSelect(rootSelect).groupBy(countQuartersAggregation).build(); + FieldWrapper countQuartersAggregation = new FieldWrapper<>( + DSL.nullif(DSL.countDistinct(functionProvider.yearQuarter(qualifiedRootSelect)), 0).as(alias), + countColumn.getName()); + + return CommonAggregationSelect.builder() + .rootSelect(rootSelect) + .groupBy( + countQuartersAggregation) + .build(); } private static CommonAggregationSelect createTwoDateColumnAggregationSelect( - Column startColumn, - Column endColumn, - String alias, - ConnectorSqlTables tables, - SqlFunctionProvider functionProvider, - StratificationFunctions stratificationFunctions) { + Column startColumn, + Column endColumn, + String alias, + ConnectorSqlTables tables, + SqlFunctionProvider functionProvider, + StratificationFunctions stratificationFunctions) { Field startDate = DSL.field(DSL.name(tables.getRootTable(), startColumn.getName()), Date.class); Field endDate = DSL.field(DSL.name(tables.getRootTable(), endColumn.getName()), Date.class); @@ -65,85 +74,116 @@ private static CommonAggregationSelect createTwoDateColumnAggregatio } private static CommonAggregationSelect sumQuarterCount( - Field quarterStart, - Field nextQuarterStart, - String alias, - ConnectorSqlTables tables, - SqlFunctionProvider functionProvider) { + Field quarterStart, + Field nextQuarterStart, + String alias, + ConnectorSqlTables tables, + SqlFunctionProvider functionProvider) { Field quarterCount = calcQuarterCount(quarterStart, nextQuarterStart, alias, functionProvider); FieldWrapper quarterCountWrapper = new FieldWrapper<>(quarterCount); - Field qualifiedQuarterCount = quarterCountWrapper.qualify(tables.cteName(ConceptCteStep.PREPROCESSING)).select(); - FieldWrapper quarterCountAggregation = new FieldWrapper<>(DSL.nullif(DSL.sum(qualifiedQuarterCount), BigDecimal.ZERO).as(alias)); + Field qualifiedQuarterCount = quarterCountWrapper.qualify( + tables.cteName(ConceptCteStep.PREPROCESSING)).select(); + FieldWrapper quarterCountAggregation = new FieldWrapper<>( + DSL.nullif(DSL.sum(qualifiedQuarterCount), BigDecimal.ZERO).as(alias)); - return CommonAggregationSelect.builder().rootSelect(quarterCountWrapper).groupBy(quarterCountAggregation).build(); + return CommonAggregationSelect.builder() + .rootSelect(quarterCountWrapper) + .groupBy( + quarterCountAggregation) + .build(); } - private static Field calcQuarterCount(Field quarterStart, Field nextQuarterStart, String alias, SqlFunctionProvider functionProvider) { - return functionProvider.dateDistance(ChronoUnit.MONTHS, quarterStart, nextQuarterStart).divide(Interval.QUARTER_INTERVAL.getAmount()).as(alias); + private static Field calcQuarterCount( + Field quarterStart, + Field nextQuarterStart, + String alias, + SqlFunctionProvider functionProvider) { + return functionProvider.dateDistance(ChronoUnit.MONTHS, quarterStart, nextQuarterStart) + .divide( + Interval.QUARTER_INTERVAL.getAmount()) + .as(alias); } private static CommonAggregationSelect buildSqlSelect( - ColumnId column, - ColumnId startColumn, - ColumnId endColumn, - String alias, - ConnectorSqlTables tables, - SqlFunctionProvider functionProvider, - StratificationFunctions stratificationFunctions) { + ColumnId column, + ColumnId startColumn, + ColumnId endColumn, + String alias, + ConnectorSqlTables tables, + SqlFunctionProvider functionProvider, + StratificationFunctions stratificationFunctions) { if (column == null) { - return createTwoDateColumnAggregationSelect(startColumn.resolve(), endColumn.resolve(), alias, tables, functionProvider, stratificationFunctions); + return createTwoDateColumnAggregationSelect( + startColumn.resolve(), + endColumn.resolve(), + alias, + tables, + functionProvider, + stratificationFunctions); } Column countColumn = column.resolve(); - return createSingleDateColumnAggregationSelect(countColumn, alias, tables, functionProvider); + return createSingleDateColumnAggregationSelect(countColumn, alias, tables, functionProvider); } @Override - public ConnectorSqlSelects connectorSelect(CountQuartersSelect countQuartersSelect, SelectContext selectContext) { - - CommonAggregationSelect countAggregationSelect = - buildSqlSelect(countQuartersSelect.getColumn(), - countQuartersSelect.getStartColumn(), - countQuartersSelect.getEndColumn(), - selectContext.getNameGenerator().selectName(countQuartersSelect), - selectContext.getTables(), - selectContext.getFunctionProvider(), - selectContext.getDialectBundle().getStratificationFunctions() - ); + public ConnectorSqlSelects connectorSelect( + CountQuartersSelect countQuartersSelect, + SelectContext selectContext) { + + CommonAggregationSelect countAggregationSelect = buildSqlSelect( + countQuartersSelect.getColumn(), + countQuartersSelect.getStartColumn(), + countQuartersSelect.getEndColumn(), + selectContext.getNameGenerator().selectName(countQuartersSelect), + selectContext.getTables(), + selectContext.getFunctionProvider(), + selectContext.getDialectBundle().getStratificationFunctions() + ); String finalPredecessor = selectContext.getTables().getPredecessor(ConceptCteStep.AGGREGATION_FILTER); - ExtractingSqlSelect finalSelect = countAggregationSelect.getGroupBy().qualify(finalPredecessor); + ExtractingSqlSelect finalSelect = countAggregationSelect.getGroupBy() + .qualify( + finalPredecessor); return ConnectorSqlSelects.builder() - .preprocessingSelects(countAggregationSelect.getRootSelects()) - .aggregationSelect(countAggregationSelect.getGroupBy()) - .finalSelect(finalSelect) - .build(); + .preprocessingSelects( + countAggregationSelect.getRootSelects()) + .aggregationSelect(countAggregationSelect.getGroupBy()) + .finalSelect( + finalSelect) + .build(); } @Override - public SqlFilters convertToSqlFilter(CountQuartersFilter countQuartersFilter, FilterContext filterContext) { - - CommonAggregationSelect countAggregationSelect = - buildSqlSelect(countQuartersFilter.getColumn(), - countQuartersFilter.getStartColumn(), - countQuartersFilter.getEndColumn(), - filterContext.getNameGenerator().selectName(countQuartersFilter), - filterContext.getTables(), - filterContext.getFunctionProvider(), - filterContext.getDialectBundle().getStratificationFunctions() - ); + public SqlFilters convertToSqlFilter( + CountQuartersFilter countQuartersFilter, + FilterContext filterContext) { + + CommonAggregationSelect countAggregationSelect = buildSqlSelect( + countQuartersFilter.getColumn(), + countQuartersFilter.getStartColumn(), + countQuartersFilter.getEndColumn(), + filterContext.getNameGenerator().selectName(countQuartersFilter), + filterContext.getTables(), + filterContext.getFunctionProvider(), + filterContext.getDialectBundle().getStratificationFunctions() + ); ConnectorSqlSelects selects = ConnectorSqlSelects.builder() - .preprocessingSelects(countAggregationSelect.getRootSelects()) - .aggregationSelect(countAggregationSelect.getGroupBy()) - .build(); + .preprocessingSelects( + countAggregationSelect.getRootSelects()) + .aggregationSelect(countAggregationSelect.getGroupBy()) + .build(); String predecessorTableName = filterContext.getTables().getPredecessor(ConceptCteStep.AGGREGATION_FILTER); - Field qualifiedCountSelect = countAggregationSelect.getGroupBy().qualify(predecessorTableName).select(); + Field qualifiedCountSelect = countAggregationSelect.getGroupBy() + .qualify( + predecessorTableName) + .select(); CountCondition countCondition = new CountCondition(qualifiedCountSelect, filterContext.getValue()); WhereClauses whereClauses = WhereClauses.builder().groupFilter(countCondition).build(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java index b1bc5222e8e..809ced14f7d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java @@ -26,40 +26,53 @@ public class CountSqlAggregator implements SelectConverter, FilterConverter, SqlAggregator { @Override - public ConnectorSqlSelects connectorSelect(CountSelect countSelect, SelectContext selectContext) { + public ConnectorSqlSelects connectorSelect( + CountSelect countSelect, + SelectContext selectContext) { ConnectorSqlTables tables = selectContext.getTables(); boolean distinct = countSelect.isDistinct(); Column countColumn = countSelect.getColumn().resolve(); String alias = selectContext.getNameGenerator().selectName(countSelect); - CommonAggregationSelect countAggregationSelect = createCountAggregationSelect(countColumn, distinct, alias, tables); + CommonAggregationSelect countAggregationSelect = createCountAggregationSelect( + countColumn, + distinct, + alias, + tables); String finalPredecessor = tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER); ExtractingSqlSelect finalSelect = countAggregationSelect.getGroupBy().qualify(finalPredecessor); return ConnectorSqlSelects.builder() - .preprocessingSelects(countAggregationSelect.getRootSelects()) - .aggregationSelect(countAggregationSelect.getGroupBy()) - .finalSelect(finalSelect) - .build(); + .preprocessingSelects( + countAggregationSelect.getRootSelects()) + .aggregationSelect(countAggregationSelect.getGroupBy()) + .finalSelect( + finalSelect) + .build(); } - private CommonAggregationSelect createCountAggregationSelect(Column countColumn, boolean distinct, String alias, ConnectorSqlTables tables) { + private CommonAggregationSelect createCountAggregationSelect( + Column countColumn, + boolean distinct, + String alias, + ConnectorSqlTables tables) { - ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(tables.getRootTable(), countColumn.getName(), Object.class); + ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>( + tables.getRootTable(), + countColumn.getName(), + Object.class); - Field qualifiedRootSelect = rootSelect.qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_SELECT)).select(); - Field countField = distinct - ? DSL.countDistinct(qualifiedRootSelect) - : DSL.count(qualifiedRootSelect); - FieldWrapper countGroupBy = new FieldWrapper<>(DSL.nullif(countField, 0).as(alias), countColumn.getName()); + Field qualifiedRootSelect = rootSelect.qualify( + tables.getPredecessor(ConceptCteStep.AGGREGATION_SELECT)).select(); + Field countField = distinct ? DSL.countDistinct(qualifiedRootSelect) : DSL.count(qualifiedRootSelect); + FieldWrapper countGroupBy = new FieldWrapper<>( + DSL.nullif(countField, 0).as(alias), + countColumn.getName()); - return CommonAggregationSelect.builder() - .rootSelect(rootSelect) - .groupBy(countGroupBy) - .build(); + return CommonAggregationSelect.builder().rootSelect(rootSelect).groupBy(countGroupBy).build(); } @Override @@ -70,17 +83,23 @@ public SqlFilters convertToSqlFilter(CountFilter countFilter, FilterContext countAggregationSelect = createCountAggregationSelect(countColumn, distinct, alias, tables); + CommonAggregationSelect countAggregationSelect = createCountAggregationSelect( + countColumn, + distinct, + alias, + tables); ConnectorSqlSelects selects = ConnectorSqlSelects.builder() - .preprocessingSelects(countAggregationSelect.getRootSelects()) - .aggregationSelect(countAggregationSelect.getGroupBy()) - .build(); - - Field qualifiedCountSelect = countAggregationSelect.getGroupBy().qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)).select(); + .preprocessingSelects( + countAggregationSelect.getRootSelects()) + .aggregationSelect(countAggregationSelect.getGroupBy()) + .build(); + + Field qualifiedCountSelect = countAggregationSelect.getGroupBy() + .qualify( + tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)) + .select(); CountCondition countCondition = new CountCondition(qualifiedCountSelect, filterContext.getValue()); - WhereClauses whereClauses = WhereClauses.builder() - .groupFilter(countCondition) - .build(); + WhereClauses whereClauses = WhereClauses.builder().groupFilter(countCondition).build(); return new SqlFilters(selects, whereClauses); } @@ -92,5 +111,4 @@ public Condition convertForTableExport(CountFilter countFilter, FilterContext, FilterConverter { @Override - public ConnectorSqlSelects connectorSelect(DateDistanceSelect select, SelectContext selectContext) { + public ConnectorSqlSelects connectorSelect( + DateDistanceSelect select, + SelectContext selectContext) { Column column = select.getColumn().resolve(); String alias = selectContext.getNameGenerator().selectName(select); ConnectorSqlTables tables = selectContext.getTables(); - Field dateDistanceCalculation = createDateDistanceCalculation(column, select.getTimeUnit(), tables, selectContext.getConversionContext()); + Field dateDistanceCalculation = createDateDistanceCalculation( + column, + select.getTimeUnit(), + tables, + selectContext.getConversionContext()); FieldWrapper dateDistanceSelect = new FieldWrapper<>(dateDistanceCalculation.as(alias)); - Field qualifiedDateDistance = dateDistanceSelect.qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_SELECT)).select(); + Field qualifiedDateDistance = dateDistanceSelect.qualify( + tables.getPredecessor(ConceptCteStep.AGGREGATION_SELECT)).select(); FieldWrapper minDateDistance = new FieldWrapper<>(DSL.min(qualifiedDateDistance).as(alias)); String finalPredecessor = tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER); ExtractingSqlSelect finalSelect = minDateDistance.qualify(finalPredecessor); return ConnectorSqlSelects.builder() - .preprocessingSelect(dateDistanceSelect) - .aggregationSelect(minDateDistance) - .finalSelect(finalSelect) - .build(); + .preprocessingSelect(dateDistanceSelect) + .aggregationSelect( + minDateDistance) + .finalSelect(finalSelect) + .build(); } @Override @@ -55,8 +63,14 @@ public SqlFilters convertToSqlFilter(DateDistanceFilter filter, FilterContext dateDistanceCalculation = createDateDistanceCalculation(column, filter.getTimeUnit(), tables, filterContext.getConversionContext()); - WhereCondition dateDistanceCondition = new DateDistanceCondition(dateDistanceCalculation, filterContext.getValue()); + Field dateDistanceCalculation = createDateDistanceCalculation( + column, + filter.getTimeUnit(), + tables, + filterContext.getConversionContext()); + WhereCondition dateDistanceCondition = new DateDistanceCondition( + dateDistanceCalculation, + filterContext.getValue()); WhereClauses whereClauses = WhereClauses.builder().eventFilter(dateDistanceCondition).build(); @@ -73,19 +87,23 @@ public Condition convertForTableExport(DateDistanceFilter filter, FilterContext< Field startDateField = DSL.field(DSL.name(tableName, columnName), Date.class); Field endDate = getEndDate(filterContext.getConversionContext()); - Field dateDistance = filterContext.getFunctionProvider().dateDistance(filter.getTimeUnit(), startDateField, endDate); + Field dateDistance = filterContext.getFunctionProvider() + .dateDistance( + filter.getTimeUnit(), + startDateField, + endDate); return new DateDistanceCondition(dateDistance, filterContext.getValue()).condition(); } private Field createDateDistanceCalculation( - Column column, - ChronoUnit timeUnit, - SqlTables tables, - ConversionContext conversionContext + Column column, + ChronoUnit timeUnit, + SqlTables tables, + ConversionContext conversionContext ) { - Field startDate = field(DSL.name(tables.getRootTable(), column.getName()), Date.class); + Field startDate = field(DSL.name(tables.getRootTable(), column.getName()), Date.class); - Field endDate = getEndDate(conversionContext); + Field endDate = getEndDate(conversionContext); SqlFunctionProvider functionProvider = conversionContext.getFunctionProvider(); return functionProvider.dateDistance(timeUnit, startDate, endDate); @@ -97,7 +115,10 @@ private Field getEndDate(ConversionContext conversionContext) { // if there is a stratification active, the upper bound of the stratification date is the end date if (conversionContext.isWithStratification()) { - ColumnDateRange stratificationDate = conversionContext.getStratificationTable().getQualifiedSelects().getStratificationDate().get(); + ColumnDateRange stratificationDate = conversionContext.getStratificationTable() + .getQualifiedSelects() + .getStratificationDate() + .get(); ColumnDateRange dualColumn = functionProvider.toDualColumn(stratificationDate); // end date is allways treated exclusive, so we get the actual end date when subtracting 1 day return functionProvider.addDays(dualColumn.getEnd(), DSL.inline(-1)); @@ -109,8 +130,7 @@ private Field getEndDate(ConversionContext conversionContext) { CDateRange dateRestriction = conversionContext.getDateRestrictionRange(); if (dateRestriction != null && dateRestriction.getMax() != null) { endDate = dateRestriction.getMax(); - } - else { + } else { // otherwise the current date is the upper bound endDate = LocalDate.now(conversionContext.getClock()); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DurationSumSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DurationSumSqlAggregator.java index 93f9c41151f..d8b6be66526 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DurationSumSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DurationSumSqlAggregator.java @@ -30,17 +30,19 @@ public class DurationSumSqlAggregator implements SelectConverter, FilterConverter, SqlAggregator { @Override - public ConnectorSqlSelects connectorSelect(DurationSumSelect select, SelectContext selectContext) { + public ConnectorSqlSelects connectorSelect( + DurationSumSelect select, + SelectContext selectContext) { return DaterangeSelectUtil.createForSelect(select, durationSumSelectFunction(), selectContext); } @Override public SqlFilters convertToSqlFilter(DurationSumFilter filter, FilterContext context) { return DaterangeSelectUtil.createForFilter( - filter, - durationSumSelectFunction(), - (aggregationField -> new SumCondition((Field) aggregationField, context.getValue())), - context); + filter, + durationSumSelectFunction(), + (aggregationField -> new SumCondition((Field) aggregationField, context.getValue())), + context); } @Override @@ -56,8 +58,7 @@ public Condition convertForTableExport(DurationSumFilter filter, FilterContext daterangeField = field(DSL.name(tableName, column.getName()), Date.class); startDateField = daterangeField; endDateField = daterangeField; - } - else { + } else { Column startColumn = filter.getStartColumn().resolve(); Column endColumn = filter.getEndColumn().resolve(); String tableName = startColumn.getTable().getName(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java index a40af4eedac..5790d669327 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java @@ -75,23 +75,25 @@ public class FlagSqlAggregator implements SelectConverter, FilterCon */ private static Map createFlagRootSelectMap(FlagSelect flagSelect, String rootTable) { return flagSelect.getFlags() - .entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - entry -> { - Column column = entry.getValue().resolve(); - Field field = field(name(rootTable, column.getName())); - return new FieldWrapper<>(field.as(column.getName()), column.getName() - ); - } - )); + .entrySet() + .stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + entry -> { + Column column = entry.getValue().resolve(); + Field field = field(name(rootTable, column.getName())); + return new FieldWrapper<>(field.as(column.getName()), column.getName() + ); + } + )); } private static FieldWrapper createFlagSelect( - String alias, - SqlTables connectorTables, - SqlFunctionProvider functionProvider, - Map flagRootSelectMap + String alias, + SqlTables connectorTables, + SqlFunctionProvider functionProvider, + Map flagRootSelectMap ) { Map> flagFieldsMap = createRootSelectReferences(connectorTables, flagRootSelectMap); @@ -114,44 +116,52 @@ private static FieldWrapper createFlagSelect( } private static Map> createRootSelectReferences( - SqlTables connectorTables, - Map flagRootSelectMap + SqlTables connectorTables, + Map flagRootSelectMap ) { - return flagRootSelectMap.entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - entry -> (Field) entry.getValue().qualify(connectorTables.getPredecessor(ConceptCteStep.AGGREGATION_SELECT)).select() - )); + return flagRootSelectMap.entrySet() + .stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + entry -> (Field) entry.getValue() + .qualify( + connectorTables.getPredecessor(ConceptCteStep.AGGREGATION_SELECT)) + .select() + )); } /** * @return Columns names of a given flags map that match the selected flags of the filter value. */ private static List getRequiredColumns(Map flags, Set selectedFlags) { - return selectedFlags.stream() - .map(flags::get) - .map(ColumnId::resolve) - .toList(); + return selectedFlags.stream().map(flags::get).map(ColumnId::resolve).toList(); } @Override public ConnectorSqlSelects connectorSelect(FlagSelect flagSelect, SelectContext selectContext) { - SqlFunctionProvider functionProvider = selectContext.getConversionContext().getDialectBundle().getFunctionProvider(); + SqlFunctionProvider functionProvider = selectContext.getConversionContext() + .getDialectBundle() + .getFunctionProvider(); SqlTables connectorTables = selectContext.getTables(); - Map rootSelects = createFlagRootSelectMap(flagSelect, connectorTables.getRootTable()); + Map rootSelects = createFlagRootSelectMap( + flagSelect, + connectorTables.getRootTable()); String alias = selectContext.getNameGenerator().selectName(flagSelect); FieldWrapper flagAggregation = createFlagSelect(alias, connectorTables, functionProvider, rootSelects); - ExtractingSqlSelect finalSelect = flagAggregation.qualify(connectorTables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)); + ExtractingSqlSelect finalSelect = flagAggregation.qualify( + connectorTables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)); return ConnectorSqlSelects.builder() - .preprocessingSelects(rootSelects.values()) - .aggregationSelect(flagAggregation) - .finalSelect(finalSelect) - .build(); + .preprocessingSelects(rootSelects.values()) + .aggregationSelect( + flagAggregation) + .finalSelect(finalSelect) + .build(); } @Override @@ -159,19 +169,18 @@ public SqlFilters convertToSqlFilter(FlagFilter flagFilter, FilterContext> rootSelects = getRequiredColumns(flagFilter.getFlags(), filterContext.getValue()) - .stream() + List> rootSelects = getRequiredColumns( + flagFilter.getFlags(), + filterContext.getValue()).stream() .map(Column::getName) - .map(columnName -> new ExtractingSqlSelect<>(rootTable, columnName, Boolean.class)) - .collect(Collectors.toList()); + .map( + columnName -> new ExtractingSqlSelect<>(rootTable, columnName, Boolean.class)) + .collect( + Collectors.toList()); - List> flagFields = rootSelects.stream() - .map(ExtractingSqlSelect::select) - .toList(); + List> flagFields = rootSelects.stream().map(ExtractingSqlSelect::select).toList(); FlagCondition flagCondition = new FlagCondition(flagFields); - WhereClauses whereClauses = WhereClauses.builder() - .eventFilter(flagCondition) - .build(); + WhereClauses whereClauses = WhereClauses.builder().eventFilter(flagCondition).build(); return new SqlFilters(ConnectorSqlSelects.none(), whereClauses); } @@ -179,10 +188,10 @@ public SqlFilters convertToSqlFilter(FlagFilter flagFilter, FilterContext> filterContext) { - List> flagFields = getRequiredColumns(filter.getFlags(), filterContext.getValue()) - .stream() - .map(column -> field(name(column.getTable().getName(), column.getName()), Boolean.class)) - .toList(); + List> flagFields = getRequiredColumns(filter.getFlags(), filterContext.getValue()).stream() + .map( + column -> field(name(column.getTable().getName(), column.getName()), Boolean.class)) + .toList(); return new FlagCondition(flagFields).condition(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java index 80c74df4fa9..67d47b3485d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java @@ -76,10 +76,7 @@ * * */ -public class SumSqlAggregator> implements - SelectConverter, - FilterConverter, RANGE>, - SqlAggregator { +public class SumSqlAggregator> implements SelectConverter, FilterConverter, RANGE>, SqlAggregator { private static final String ROW_NUMBER_ALIAS = "row_number"; private static final String SUM_DISTINCT_SUFFIX = "sum_distinct"; @@ -101,68 +98,98 @@ public ConnectorSqlSelects connectorSelect(SumSelect sumSelect, SelectContext finalSelect = createFinalSelect(sumAggregationSelect, tables); return ConnectorSqlSelects.builder() - .preprocessingSelects(sumAggregationSelect.getRootSelects()) - .additionalPredecessor(sumAggregationSelect.getAdditionalPredecessor()) - .finalSelect(finalSelect) - .build(); - } - else { + .preprocessingSelects( + sumAggregationSelect.getRootSelects()) + .additionalPredecessor( + sumAggregationSelect.getAdditionalPredecessor()) + .finalSelect(finalSelect) + .build(); + } else { sumAggregationSelect = createSumAggregationSelect(sumColumn, subtractColumn, alias, tables); ExtractingSqlSelect finalSelect = createFinalSelect(sumAggregationSelect, tables); return ConnectorSqlSelects.builder() - .preprocessingSelects(sumAggregationSelect.getRootSelects()) - .aggregationSelect(sumAggregationSelect.getGroupBy()) - .finalSelect(finalSelect) - .build(); + .preprocessingSelects( + sumAggregationSelect.getRootSelects()) + .aggregationSelect(sumAggregationSelect.getGroupBy()) + .finalSelect( + finalSelect) + .build(); } } private CommonAggregationSelect createDistinctSumAggregationSelect( - Column sumColumn, - List distinctByColumns, - String alias, - SqlIdColumns ids, - ConnectorSqlTables tables, - NameGenerator nameGenerator + Column sumColumn, + List distinctByColumns, + String alias, + SqlIdColumns ids, + ConnectorSqlTables tables, + NameGenerator nameGenerator ) { List> preprocessingSelects = new ArrayList<>(); Class numberClass = NumberMapUtil.getType(sumColumn); - ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(tables.getRootTable(), sumColumn.getName(), numberClass); + ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>( + tables.getRootTable(), + sumColumn.getName(), + numberClass); preprocessingSelects.add(rootSelect); - List> distinctByRootSelects = - distinctByColumns.stream() - .map(column -> new ExtractingSqlSelect<>(tables.getRootTable(), column.getName(), Object.class)) - .collect(Collectors.toList()); + List> distinctByRootSelects = distinctByColumns.stream() + .map( + column -> new ExtractingSqlSelect<>(tables.getRootTable(), column.getName(), Object.class)) + .collect( + Collectors.toList()); preprocessingSelects.addAll(distinctByRootSelects); - QueryStep rowNumberCte = createRowNumberCte(ids, rootSelect, distinctByRootSelects, alias, tables, nameGenerator); + QueryStep rowNumberCte = createRowNumberCte( + ids, + rootSelect, + distinctByRootSelects, + alias, + tables, + nameGenerator); Field rootSelectQualified = rootSelect.qualify(rowNumberCte.getCteName()).select(); - FieldWrapper sumGroupBy = new FieldWrapper<>(DSL.sum(DSL.coalesce(rootSelectQualified, DSL.inline(0))).as(alias)); + FieldWrapper sumGroupBy = new FieldWrapper<>( + DSL.sum(DSL.coalesce(rootSelectQualified, DSL.inline(0))).as(alias)); QueryStep rowNumberFilteredCte = createRowNumberFilteredCte(rowNumberCte, sumGroupBy, alias, nameGenerator); return CommonAggregationSelect.builder() - .rootSelects(preprocessingSelects) - .additionalPredecessor(rowNumberFilteredCte) - .groupBy(sumGroupBy) - .build(); + .rootSelects(preprocessingSelects) + .additionalPredecessor( + rowNumberFilteredCte) + .groupBy(sumGroupBy) + .build(); } - private static ExtractingSqlSelect createFinalSelect(CommonAggregationSelect sumAggregationSelect, ConnectorSqlTables tables) { + private static ExtractingSqlSelect createFinalSelect( + CommonAggregationSelect sumAggregationSelect, + ConnectorSqlTables tables) { String finalPredecessor = tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER); return sumAggregationSelect.getGroupBy().qualify(finalPredecessor); } - private CommonAggregationSelect createSumAggregationSelect(Column sumColumn, Column subtractColumn, String alias, ConnectorSqlTables tables) { + private CommonAggregationSelect createSumAggregationSelect( + Column sumColumn, + Column subtractColumn, + String alias, + ConnectorSqlTables tables) { Class numberClass = NumberMapUtil.getType(sumColumn); List> preprocessingSelects = new ArrayList<>(); - ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(tables.getRootTable(), sumColumn.getName(), numberClass); + ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>( + tables.getRootTable(), + sumColumn.getName(), + numberClass); preprocessingSelects.add(rootSelect); String preprocessingCte = tables.cteName(ConceptCteStep.PREPROCESSING); @@ -173,9 +200,9 @@ private CommonAggregationSelect createSumAggregationSelect(Column su if (subtractColumn != null) { ExtractingSqlSelect subtractColumnRootSelect = new ExtractingSqlSelect<>( - tables.getRootTable(), - subtractColumn.getName(), - numberClass + tables.getRootTable(), + subtractColumn.getName(), + numberClass ); preprocessingSelects.add(subtractColumnRootSelect); @@ -186,19 +213,23 @@ private CommonAggregationSelect createSumAggregationSelect(Column su // COALESCE would always result in 0 which is undesired to differentiate between missing-values and 0 sums. Field zeroIfAnyNonNull = DSL.coalesce(sumField.multiply(0), subtractField.multiply(0)); - sumGroupBy = new FieldWrapper<>(DSL.sum(DSL.coalesce(sumField, zeroIfAnyNonNull).minus(DSL.coalesce(subtractField, zeroIfAnyNonNull))).as(alias), - sumColumn.getName(), - subtractColumn.getName() + sumGroupBy = new FieldWrapper<>( + DSL.sum( + DSL.coalesce(sumField, zeroIfAnyNonNull).minus(DSL.coalesce(subtractField, zeroIfAnyNonNull))) + .as( + alias), + sumColumn.getName(), + subtractColumn.getName() ); - } - else { + } else { sumGroupBy = new FieldWrapper<>(DSL.sum(sumField).as(alias), sumColumn.getName()); } return CommonAggregationSelect.builder() - .rootSelects(preprocessingSelects) - .groupBy(sumGroupBy) - .build(); + .rootSelects(preprocessingSelects) + .groupBy( + sumGroupBy) + .build(); } /** @@ -206,66 +237,67 @@ private CommonAggregationSelect createSumAggregationSelect(Column su * the row number will be incremented for each duplicated entry. */ private static QueryStep createRowNumberCte( - SqlIdColumns ids, - SingleColumnSqlSelect sumColumnRootSelect, - List> distinctByRootSelects, - String alias, - SqlTables connectorTables, - NameGenerator nameGenerator + SqlIdColumns ids, + SingleColumnSqlSelect sumColumnRootSelect, + List> distinctByRootSelects, + String alias, + SqlTables connectorTables, + NameGenerator nameGenerator ) { String predecessor = connectorTables.getPredecessor(ConceptCteStep.AGGREGATION_SELECT); SqlIdColumns qualifiedIds = ids.qualify(predecessor); SingleColumnSqlSelect qualifiedSumRootSelect = sumColumnRootSelect.qualify(predecessor); List> partitioningFields = Stream.concat( - qualifiedIds.toFields().stream(), - distinctByRootSelects.stream().map(sqlSelect -> sqlSelect.qualify(predecessor).select()) - ) - .collect(Collectors.toList()); + qualifiedIds.toFields().stream(), + distinctByRootSelects.stream().map(sqlSelect -> sqlSelect.qualify(predecessor).select()) + ).collect(Collectors.toList()); FieldWrapper rowNumber = new FieldWrapper<>( - DSL.rowNumber().over(DSL.partitionBy(partitioningFields)).as(ROW_NUMBER_ALIAS), - partitioningFields.stream().map(Field::getName).toArray(String[]::new) + DSL.rowNumber().over(DSL.partitionBy(partitioningFields)).as(ROW_NUMBER_ALIAS), + partitioningFields.stream().map(Field::getName).toArray(String[]::new) ); Selects rowNumberAssignedSelects = Selects.builder() - .ids(qualifiedIds) - .sqlSelects(List.of(qualifiedSumRootSelect, rowNumber)) - .build(); + .ids(qualifiedIds) + .sqlSelects( + List.of(qualifiedSumRootSelect, rowNumber)) + .build(); return QueryStep.builder() - .cteName(nameGenerator.cteStepName(SumDistinctCteStep.ROW_NUMBER_ASSIGNED, alias)) - .selects(rowNumberAssignedSelects) - .fromTable(QueryStep.toTableLike(predecessor)) - .build(); + .cteName( + nameGenerator.cteStepName(SumDistinctCteStep.ROW_NUMBER_ASSIGNED, alias)) + .selects( + rowNumberAssignedSelects) + .fromTable(QueryStep.toTableLike(predecessor)) + .build(); } /** * Sums up the sum column values but only those whose row number is 1. Thus, only unique entries will be summed up. */ private static QueryStep createRowNumberFilteredCte( - QueryStep rowNumberCte, - FieldWrapper sumSelect, - String alias, - NameGenerator nameGenerator + QueryStep rowNumberCte, + FieldWrapper sumSelect, + String alias, + NameGenerator nameGenerator ) { SqlIdColumns ids = rowNumberCte.getQualifiedSelects().getIds(); - Selects rowNumberFilteredSelects = Selects.builder() - .ids(ids) - .sqlSelects(List.of(sumSelect)) - .build(); + Selects rowNumberFilteredSelects = Selects.builder().ids(ids).sqlSelects(List.of(sumSelect)).build(); - Condition firstOccurrence = DSL.field(DSL.name(rowNumberCte.getCteName(), ROW_NUMBER_ALIAS)) - .eq(DSL.inline(1)); + Condition firstOccurrence = DSL.field(DSL.name(rowNumberCte.getCteName(), ROW_NUMBER_ALIAS)).eq(DSL.inline(1)); return QueryStep.builder() - .cteName(nameGenerator.cteStepName(SumDistinctCteStep.ROW_NUMBER_FILTERED, alias)) - .selects(rowNumberFilteredSelects) - .fromTable(QueryStep.toTableLike(rowNumberCte.getCteName())) - .conditions(List.of(firstOccurrence)) - .predecessors(List.of(rowNumberCte)) - .groupBy(ids.toFields()) - .build(); + .cteName( + nameGenerator.cteStepName(SumDistinctCteStep.ROW_NUMBER_FILTERED, alias)) + .selects( + rowNumberFilteredSelects) + .fromTable(QueryStep.toTableLike(rowNumberCte.getCteName())) + .conditions( + List.of(firstOccurrence)) + .predecessors(List.of(rowNumberCte)) + .groupBy(ids.toFields()) + .build(); } @Override @@ -281,27 +313,37 @@ public SqlFilters convertToSqlFilter(SumFilter sumFilter, FilterContext qualifiedSumSelect = sumAggregationSelect.getGroupBy().qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)).select(); + Field qualifiedSumSelect = sumAggregationSelect.getGroupBy() + .qualify( + tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)) + .select(); SumCondition sumCondition = new SumCondition(qualifiedSumSelect, filterContext.getValue()); - WhereClauses whereClauses = WhereClauses.builder() - .groupFilter(sumCondition) - .build(); + WhereClauses whereClauses = WhereClauses.builder().groupFilter(sumCondition).build(); return new SqlFilters(selects, whereClauses); @@ -332,8 +374,9 @@ public Condition convertForTableExport(SumFilter filter, FilterContext, T> imple public SqlFilters convertToSqlFilter(F filter, FilterContext filterContext) { ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>( - filterContext.getTables().getPredecessor(ConceptCteStep.PREPROCESSING), - filter.getColumn().getColumn(), - String.class + filterContext.getTables().getPredecessor(ConceptCteStep.PREPROCESSING), + filter.getColumn().getColumn(), + String.class ); WhereCondition condition = new MultiSelectCondition( - rootSelect.select(), - getValues(filterContext), - filterContext.getFunctionProvider() + rootSelect.select(), + getValues(filterContext), + filterContext.getFunctionProvider() ); return new SqlFilters( - ConnectorSqlSelects.none(), - WhereClauses.builder() - .eventFilter(condition) - .build() + ConnectorSqlSelects.none(), + WhereClauses.builder().eventFilter(condition).build() ); } @@ -41,7 +39,10 @@ public Condition convertForTableExport(F filter, FilterContext filterContext) String tableName = column.getTable().getName(); String columnName = column.getName(); Field field = DSL.field(DSL.name(tableName, columnName), String.class); - return new MultiSelectCondition(field, getValues(filterContext), filterContext.getFunctionProvider()).condition(); + return new MultiSelectCondition( + field, + getValues(filterContext), + filterContext.getFunctionProvider()).condition(); } protected abstract String[] getValues(FilterContext filterContext); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/ConditionUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/ConditionUtil.java index 2cb8fe04f0a..d39cde0180e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/ConditionUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/ConditionUtil.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.sql.conversion.model.filter; -import java.util.Optional; -import java.util.stream.Stream; import com.bakdata.conquery.models.common.IRange; import lombok.experimental.UtilityClass; @@ -15,11 +13,11 @@ public class ConditionUtil { public > Condition rangeCondition(final Field column, final IRange range) { Condition condition = DSL.noCondition(); - if (range.hasLowerBound()){ + if (range.hasLowerBound()) { condition = condition.and(column.greaterOrEqual(DSL.inline(range.getMin()))); } - if (range.hasUpperBound()){ + if (range.hasUpperBound()) { condition = condition.and(column.lessOrEqual(DSL.inline(range.getMax()))); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/FlagCondition.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/FlagCondition.java index 9a1d24fb06b..e2ffd22085f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/FlagCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/FlagCondition.java @@ -15,10 +15,11 @@ public class FlagCondition implements WhereCondition { @Override public Condition condition() { return flagFields.stream() - .map(DSL::condition) - .map(Field::isTrue) - .reduce(Condition::or) - .orElseThrow(() -> new IllegalArgumentException("Can't construct a FlagCondition with an empty flag field list.")); + .map(DSL::condition) + .map(Field::isTrue) + .reduce(Condition::or) + .orElseThrow( + () -> new IllegalArgumentException("Can't construct a FlagCondition with an empty flag field list.")); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/MultiSelectCondition.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/MultiSelectCondition.java index 9d778877c7f..bb6bf94daf3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/MultiSelectCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/MultiSelectCondition.java @@ -1,15 +1,15 @@ package com.bakdata.conquery.sql.conversion.model.filter; +import static org.jooq.impl.DSL.*; + +import java.util.Arrays; + import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.google.common.base.Strings; import lombok.RequiredArgsConstructor; import org.jooq.Condition; import org.jooq.Field; -import java.util.Arrays; - -import static org.jooq.impl.DSL.*; - @RequiredArgsConstructor public class MultiSelectCondition implements WhereCondition { @@ -31,8 +31,9 @@ public Condition condition() { // values can contain empty or null Strings String[] valuesWithoutNull = Arrays.stream(values) - .filter(value -> !Strings.isNullOrEmpty(value)) - .toArray(String[]::new); + .filter(value -> !Strings.isNullOrEmpty(value)) + .toArray( + String[]::new); Condition inCondition; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java index f4e24084a20..67177f294b2 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java @@ -21,7 +21,10 @@ public SqlFilters convertToSqlFilter(NumberFilter filter, FilterContext numberClass = NumberMapUtil.getType(column); - ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(tables.getRootTable(), column.getName(), numberClass); + ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>( + tables.getRootTable(), + column.getName(), + numberClass); IRange filterValue = filterContext.getValue(); NumberCondition condition = new NumberCondition(rootSelect.select(), filterValue); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/RangeCondition.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/RangeCondition.java index bd537efb939..215acf73e11 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/RangeCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/RangeCondition.java @@ -9,7 +9,7 @@ abstract class RangeCondition> implements WhereCondition { private final Field column; - private final IRange range; + private final IRange range; @Override public Condition condition() { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/WhereCondition.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/WhereCondition.java index d5c4cb98858..dd7b7f9abbd 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/WhereCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/WhereCondition.java @@ -20,7 +20,9 @@ default WhereCondition or(WhereCondition whereCondition) { return combineConditions(whereCondition, Condition::or); } - private WhereCondition combineConditions(WhereCondition whereCondition, BiFunction operation) { + private WhereCondition combineConditions( + WhereCondition whereCondition, + BiFunction operation) { Condition combinedCondition = operation.apply(this.condition(), whereCondition.condition()); return new ConditionWrappingWhereCondition(combinedCondition); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java index 698cd1f472e..43647f97281 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java @@ -31,22 +31,24 @@ public class ConceptColumnSelectConverter implements SelectConverter selectContext) { + public ConnectorSqlSelects connectorSelect( + ConceptColumnSelect select, + SelectContext selectContext) { Connector connector = selectContext.getTables().getConnector(); if (connector.getColumn() == null) { return ConnectorSqlSelects.none(); } - ExtractingSqlSelect connectorColumn = new ExtractingSqlSelect<>(connector.resolveTableId().getTable(), connector.getColumn().getColumn(), Object.class); - return ConnectorSqlSelects.builder() - .preprocessingSelect(connectorColumn) - .build(); + ExtractingSqlSelect connectorColumn = new ExtractingSqlSelect<>( + connector.resolveTableId().getTable(), + connector.getColumn().getColumn(), + Object.class); + return ConnectorSqlSelects.builder().preprocessingSelect(connectorColumn).build(); } @Override @@ -58,9 +60,13 @@ public ConceptSqlSelects conceptSelect(ConceptColumnSelect select, SelectContext // we union the Connector table with itself if there is only 1 Connector Connector connector = selectContext.getTables().getConnectorTables().get(0).getConnector(); connectors = List.of(connector, connector); - } - else { - connectors = selectContext.getTables().getConnectorTables().stream().map(ConnectorSqlTables::getConnector).toList(); + } else { + connectors = selectContext.getTables() + .getConnectorTables() + .stream() + .map( + ConnectorSqlTables::getConnector) + .toList(); } NameGenerator nameGenerator = selectContext.getNameGenerator(); @@ -69,26 +75,26 @@ public ConceptSqlSelects conceptSelect(ConceptColumnSelect select, SelectContext FieldWrapper conceptColumnSelect = createConnectorColumnStringAgg(selectContext, unionStep, alias); Selects unionStepSelects = unionStep.getQualifiedSelects(); - Selects selects = Selects.builder() - .ids(unionStepSelects.getIds()) - .sqlSelect(conceptColumnSelect) - .build(); + Selects selects = Selects.builder().ids(unionStepSelects.getIds()).sqlSelect(conceptColumnSelect).build(); String stringAggCteName = nameGenerator.cteStepName(CONCEPT_COLUMN_STEPS.STRING_AGG, alias); QueryStep stringAggStep = QueryStep.builder() - .cteName(stringAggCteName) - .selects(selects) - .fromTable(QueryStep.toTableLike(unionStep.getCteName())) - .groupBy(unionStepSelects.getIds().toFields()) - .predecessor(unionStep) - .build(); + .cteName(stringAggCteName) + .selects(selects) + .fromTable( + QueryStep.toTableLike(unionStep.getCteName())) + .groupBy(unionStepSelects.getIds().toFields()) + .predecessor( + unionStep) + .build(); ExtractingSqlSelect finalSelect = conceptColumnSelect.qualify(stringAggStep.getCteName()); return ConceptSqlSelects.builder() - .additionalPredecessor(Optional.of(stringAggStep)) - .finalSelect(finalSelect) - .build(); + .additionalPredecessor(Optional.of(stringAggStep)) + .finalSelect( + finalSelect) + .build(); } private static boolean isSingleConnector(ConceptSqlTables tables) { @@ -96,52 +102,66 @@ private static boolean isSingleConnector(ConceptSqlTables tables) { } private static QueryStep createUnionConnectorConnectorsStep( - List connectors, - String alias, - SelectContext selectContext + List connectors, + String alias, + SelectContext selectContext ) { - List unionSteps = connectors.stream().map(connector -> createConnectorColumnSelectQuery(connector, alias, selectContext)).toList(); - String unionedColumnsCteName = selectContext.getNameGenerator().cteStepName(CONCEPT_COLUMN_STEPS.UNIONED_COLUMNS, alias); - return QueryStep.createUnionStep(unionSteps, unionedColumnsCteName, Collections.emptyList(), false, selectContext.getFunctionProvider()); //TODO is false correct here? + List unionSteps = connectors.stream() + .map( + connector -> createConnectorColumnSelectQuery(connector, alias, selectContext)) + .toList(); + String unionedColumnsCteName = selectContext.getNameGenerator() + .cteStepName( + CONCEPT_COLUMN_STEPS.UNIONED_COLUMNS, + alias); + return QueryStep.createUnionStep( + unionSteps, + unionedColumnsCteName, + Collections.emptyList(), + false, + selectContext.getFunctionProvider()); //TODO is false correct here? } private static QueryStep createConnectorColumnSelectQuery( - Connector connector, - String alias, - SelectContext selectContext + Connector connector, + String alias, + SelectContext selectContext ) { // a ConceptColumn select uses all connectors a Concept has, even if they are not part of the CQConcept // but if they are, we need to make sure we use the preprocessed and event-filtered table instead of the root table String tableName = selectContext.getTables() - .getConnectorTables() - .stream() - .filter(tables -> Objects.equals(tables.getRootTable(), connector.resolveTableId().getTable())) - .findFirst() - .map(tables -> tables.cteName(ConceptCteStep.PREPROCESSING)) - .orElse(connector.resolveTableId().getTable()); + .getConnectorTables() + .stream() + .filter( + tables -> Objects.equals(tables.getRootTable(), connector.resolveTableId().getTable())) + .findFirst() + .map( + tables -> tables.cteName(ConceptCteStep.PREPROCESSING)) + .orElse(connector.resolveTableId().getTable()); Table connectorTable = DSL.table(DSL.name(tableName)); SqlIdColumns ids = selectContext.getIds().qualify(connectorTable.getName()); - Field connectorColumn = DSL.field(DSL.name(connectorTable.getName(), connector.getColumn().resolve().getName())); + Field connectorColumn = DSL.field( + DSL.name(connectorTable.getName(), connector.getColumn().resolve().getName())); Field casted = selectContext.getFunctionProvider().cast(connectorColumn, SQLDataType.VARCHAR).as(alias); FieldWrapper connectorSelect = new FieldWrapper<>(casted); - Selects selects = Selects.builder() - .ids(ids) - .sqlSelect(connectorSelect) - .build(); + Selects selects = Selects.builder().ids(ids).sqlSelect(connectorSelect).build(); - return QueryStep.builder() - .selects(selects) - .fromTable(connectorTable) - .build(); + return QueryStep.builder().selects(selects).fromTable(connectorTable).build(); } - private static FieldWrapper createConnectorColumnStringAgg(SelectContext selectContext, QueryStep unionStep, String alias) { + private static FieldWrapper createConnectorColumnStringAgg( + SelectContext selectContext, + QueryStep unionStep, + String alias) { SqlFunctionProvider functionProvider = selectContext.getFunctionProvider(); Field unionedColumn = DSL.field(DSL.name(unionStep.getCteName(), alias), String.class); return new FieldWrapper<>( - functionProvider.stringAggregation(unionedColumn, DSL.toChar(ResultSetProcessor.UNIT_SEPARATOR), List.of(unionedColumn)).as(alias) + functionProvider.stringAggregation( + unionedColumn, + DSL.toChar(ResultSetProcessor.UNIT_SEPARATOR), + List.of(unionedColumn)).as(alias) ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java index a03f8534836..57daa1e36a6 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java @@ -6,11 +6,14 @@ public class DateUnionSelectConverter implements SelectConverter { @Override - public ConnectorSqlSelects connectorSelect(DateUnionSelect select, SelectContext selectContext) { + public ConnectorSqlSelects connectorSelect( + DateUnionSelect select, + SelectContext selectContext) { return DaterangeSelectUtil.createForSelect( - select, - (daterange, alias, functionProvider) -> new FieldWrapper<>(functionProvider.dateRangeAggregation(daterange).as(alias)), - selectContext + select, + (daterange, alias, functionProvider) -> new FieldWrapper<>( + functionProvider.dateRangeAggregation(daterange).as(alias)), + selectContext ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java index 005be1cd75c..cf1e99c2b60 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java @@ -44,40 +44,45 @@ public class DaterangeSelectUtil { * {@link IntervalPackingSelectsCte}s using additional predecessor tables. */ public static ConnectorSqlSelects createForSelect( - S select, - AggregationFunction aggregationFunction, - SelectContext context + S select, + AggregationFunction aggregationFunction, + SelectContext context ) { String alias = context.getNameGenerator().selectName(select); SqlFunctionProvider functionProvider = context.getFunctionProvider(); ColumnDateRange daterange = functionProvider.forArbitraryDateRange(select).as(alias); - List rootSelects = daterange.toFields().stream() - .map(FieldWrapper::new) - .collect(Collectors.toList()); + List rootSelects = daterange.toFields().stream().map(FieldWrapper::new).collect(Collectors.toList()); SqlTables daterangeSelectTables = createTables(alias, context.getTables(), context); - QueryStep lastIntervalPackingStep = applyIntervalPacking(daterange, daterangeSelectTables, context.getIds(), context.getTables(), context.getDialectBundle()); + QueryStep lastIntervalPackingStep = applyIntervalPacking( + daterange, + daterangeSelectTables, + context.getIds(), + context.getTables(), + context.getDialectBundle()); ColumnDateRange qualified = daterange.qualify(daterangeSelectTables.getPredecessor(INTERVAL_PACKING_SELECTS)); FieldWrapper aggregationField = aggregationFunction.apply(qualified, alias, functionProvider); QueryStep intervalPackingSelectsStep = IntervalPackingSelectsCte.forSelect( - lastIntervalPackingStep, - qualified, - aggregationField, - daterangeSelectTables, - context.getDialectBundle() + lastIntervalPackingStep, + qualified, + aggregationField, + daterangeSelectTables, + context.getDialectBundle() ); ConnectorSqlTables tables = context.getTables(); - ExtractingSqlSelect finalSelect = aggregationField.qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)); + ExtractingSqlSelect finalSelect = aggregationField.qualify( + tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)); return ConnectorSqlSelects.builder() - .preprocessingSelects(rootSelects) - .additionalPredecessor(Optional.of(intervalPackingSelectsStep)) - .finalSelect(finalSelect) - .build(); + .preprocessingSelects(rootSelects) + .additionalPredecessor( + Optional.of(intervalPackingSelectsStep)) + .finalSelect(finalSelect) + .build(); } /** @@ -85,51 +90,65 @@ public static ConnectorSqlSelects c * {@link IntervalPackingSelectsCte}s using additional predecessor tables. Finally, the filter condition is created. */ public static SqlFilters createForFilter( - DaterangeSelectOrFilter filter, - AggregationFunction aggregationFunction, - Function, WhereCondition> filterFunction, - FilterContext context + DaterangeSelectOrFilter filter, + AggregationFunction aggregationFunction, + Function, WhereCondition> filterFunction, + FilterContext context ) { String alias = context.getNameGenerator().selectName((LabeledNamespaceIdentifiable) filter); SqlFunctionProvider functionProvider = context.getDialectBundle().getFunctionProvider(); ColumnDateRange daterange = functionProvider.forArbitraryDateRange(filter).as(alias); - List rootSelects = daterange.toFields().stream() - .map(FieldWrapper::new) - .collect(Collectors.toList()); + List rootSelects = daterange.toFields().stream().map(FieldWrapper::new).collect(Collectors.toList()); SqlTables daterangeSelectTables = createTables(alias, context.getTables(), context); - QueryStep lastIntervalPackingStep = applyIntervalPacking(daterange, daterangeSelectTables, context.getIds(), context.getTables(), context.getDialectBundle()); + QueryStep lastIntervalPackingStep = applyIntervalPacking( + daterange, + daterangeSelectTables, + context.getIds(), + context.getTables(), + context.getDialectBundle()); ColumnDateRange qualified = daterange.qualify(daterangeSelectTables.getPredecessor(INTERVAL_PACKING_SELECTS)); FieldWrapper aggregationField = aggregationFunction.apply(qualified, alias, functionProvider); QueryStep intervalPackingSelectsStep = IntervalPackingSelectsCte.forSelect( - lastIntervalPackingStep, - qualified, - aggregationField, - daterangeSelectTables, - context.getDialectBundle() + lastIntervalPackingStep, + qualified, + aggregationField, + daterangeSelectTables, + context.getDialectBundle() ); ConnectorSqlSelects sqlSelects = ConnectorSqlSelects.builder() - .preprocessingSelects(rootSelects) - .additionalPredecessor(Optional.of(intervalPackingSelectsStep)) - .build(); + .preprocessingSelects( + rootSelects) + .additionalPredecessor(Optional.of(intervalPackingSelectsStep)) + .build(); ConnectorSqlTables tables = context.getTables(); - Field qualifiedAggregationField = aggregationField.qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)).select(); - WhereClauses whereClauses = WhereClauses.builder().groupFilter(filterFunction.apply(qualifiedAggregationField)).build(); + Field qualifiedAggregationField = aggregationField.qualify( + tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)).select(); + WhereClauses whereClauses = WhereClauses.builder() + .groupFilter( + filterFunction.apply(qualifiedAggregationField)) + .build(); return new SqlFilters(sqlSelects, whereClauses); } - public static FieldWrapper createDurationSumSqlSelect(String alias, ColumnDateRange validityDate, SqlFunctionProvider functionProvider) { - Field dateDistanceInDays = functionProvider.dateDistance(ChronoUnit.DAYS, validityDate.getStart(), validityDate.getEnd()); - Field durationSum = sum(when(containsInfinityDate(validityDate, functionProvider), inline(null, Integer.class)) - .otherwise(dateDistanceInDays) - ) - .as(alias); + public static FieldWrapper createDurationSumSqlSelect( + String alias, + ColumnDateRange validityDate, + SqlFunctionProvider functionProvider) { + Field dateDistanceInDays = functionProvider.dateDistance( + ChronoUnit.DAYS, + validityDate.getStart(), + validityDate.getEnd()); + Field durationSum = sum( + when(containsInfinityDate(validityDate, functionProvider), inline(null, Integer.class)).otherwise( + dateDistanceInDays) + ).as(alias); return new FieldWrapper<>(durationSum); } @@ -147,31 +166,33 @@ private static SqlTables createTables(String alias, ConnectorSqlTables connector if (context.getDialectBundle().supportsSingleColumnRanges()) { predecessorMapping.put(UNNEST_DATE, INTERVAL_COMPLETE); predecessorMapping.put(INTERVAL_PACKING_SELECTS, UNNEST_DATE); - } - else { + } else { predecessorMapping.put(INTERVAL_PACKING_SELECTS, INTERVAL_COMPLETE); } - Map cteNameMap = CteStep.createCteNameMap(predecessorMapping.keySet(), alias, context.getNameGenerator()); + Map cteNameMap = CteStep.createCteNameMap( + predecessorMapping.keySet(), + alias, + context.getNameGenerator()); return new SqlTables(preprocessingCteName, cteNameMap, predecessorMapping); } private static QueryStep applyIntervalPacking( - ColumnDateRange daterange, - SqlTables dateUnionTables, - SqlIdColumns idColumns, - ConnectorSqlTables connectorSqlTables, - DialectBundle sqlDialect + ColumnDateRange daterange, + SqlTables dateUnionTables, + SqlIdColumns idColumns, + ConnectorSqlTables connectorSqlTables, + DialectBundle sqlDialect ) { String preprocessingCteName = connectorSqlTables.cteName(PREPROCESSING); IntervalPackingContext intervalPackingContext = IntervalPackingContext.builder() - .ids(idColumns.qualify(preprocessingCteName)) - .daterange(daterange.qualify(preprocessingCteName)) - .tables(dateUnionTables) - .build(); - - return sqlDialect - .getIntervalPacker() - .aggregateAsArbitrarySelect(intervalPackingContext); + .ids( + idColumns.qualify(preprocessingCteName)) + .daterange(daterange.qualify(preprocessingCteName)) + .tables( + dateUnionTables) + .build(); + + return sqlDialect.getIntervalPacker().aggregateAsArbitrarySelect(intervalPackingContext); } @FunctionalInterface diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java index c875bc4b320..601375bd8ce 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java @@ -1,8 +1,5 @@ package com.bakdata.conquery.sql.conversion.model.select; -import static org.jooq.impl.DSL.*; - -import java.util.Collections; import java.util.List; import java.util.Optional; @@ -11,7 +8,6 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.CteStep; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; @@ -19,8 +15,6 @@ import com.bakdata.conquery.sql.execution.ResultSetProcessor; import lombok.Getter; import lombok.RequiredArgsConstructor; -import org.jetbrains.annotations.NotNull; -import org.jooq.Condition; import org.jooq.Field; import org.jooq.impl.DSL; import org.jooq.impl.SQLDataType; @@ -59,64 +53,75 @@ public class DistinctSelectConverter implements SelectConverter @RequiredArgsConstructor private enum DistinctSelectCteStep implements CteStep { - DISTINCT_SELECT("distinct", null), - STRING_AGG("aggregated", DISTINCT_SELECT); + DISTINCT_SELECT("distinct", null), STRING_AGG("aggregated", DISTINCT_SELECT); private final String suffix; private final DistinctSelectCteStep predecessor; } @Override - public ConnectorSqlSelects connectorSelect(DistinctSelect distinctSelect, SelectContext selectContext) { + public ConnectorSqlSelects connectorSelect( + DistinctSelect distinctSelect, + SelectContext selectContext) { String alias = selectContext.getNameGenerator().selectName(distinctSelect); ConnectorSqlTables tables = selectContext.getTables(); - SingleColumnSqlSelect preprocessingSelect = MappableSingleColumnSelect.getSubstringSelect(distinctSelect.getColumn().get(), distinctSelect.getSubstringRange(), selectContext, alias); + SingleColumnSqlSelect preprocessingSelect = MappableSingleColumnSelect.getSubstringSelect( + distinctSelect.getColumn().get(), + distinctSelect.getSubstringRange(), + selectContext, + alias); QueryStep distinctSelectCte = createDistinctSelectCte(preprocessingSelect, alias, selectContext); QueryStep aggregatedCte = createAggregationCte(selectContext, preprocessingSelect, distinctSelectCte, alias); - SingleColumnSqlSelect finalSelect = preprocessingSelect.qualify(tables.cteName(ConceptCteStep.AGGREGATION_FILTER)); + SingleColumnSqlSelect finalSelect = preprocessingSelect.qualify( + tables.cteName(ConceptCteStep.AGGREGATION_FILTER)); return ConnectorSqlSelects.builder() - .preprocessingSelect(preprocessingSelect) - .additionalPredecessor(Optional.of(aggregatedCte)) - .finalSelect(finalSelect) - .build(); + .preprocessingSelect(preprocessingSelect) + .additionalPredecessor( + Optional.of(aggregatedCte)) + .finalSelect(finalSelect) + .build(); } private static QueryStep createAggregationCte( - SelectContext selectContext, - SingleColumnSqlSelect preprocessingSelect, - QueryStep distinctSelectCte, - String alias + SelectContext selectContext, + SingleColumnSqlSelect preprocessingSelect, + QueryStep distinctSelectCte, + String alias ) { SqlFunctionProvider functionProvider = selectContext.getFunctionProvider(); - Field castedColumn = functionProvider.cast(preprocessingSelect.qualify(distinctSelectCte.getCteName()).select(), SQLDataType.VARCHAR); - Field aggregatedColumn = functionProvider.stringAggregation(castedColumn, DSL.toChar(ResultSetProcessor.UNIT_SEPARATOR), List.of(castedColumn)) - .as(alias); + Field castedColumn = functionProvider.cast( + preprocessingSelect.qualify(distinctSelectCte.getCteName()).select(), + SQLDataType.VARCHAR); + Field aggregatedColumn = functionProvider.stringAggregation( + castedColumn, + DSL.toChar(ResultSetProcessor.UNIT_SEPARATOR), + List.of(castedColumn)).as(alias); SqlIdColumns ids = distinctSelectCte.getQualifiedSelects().getIds(); - Selects selects = Selects.builder() - .ids(ids) - .sqlSelect(new FieldWrapper<>(aggregatedColumn)) - .build(); + Selects selects = Selects.builder().ids(ids).sqlSelect(new FieldWrapper<>(aggregatedColumn)).build(); return QueryStep.builder() - .cteName(selectContext.getNameGenerator().cteStepName(DistinctSelectCteStep.STRING_AGG, alias)) - .selects(selects) - .fromTable(QueryStep.toTableLike(distinctSelectCte.getCteName())) - .groupBy(ids.toFields()) - .predecessor(distinctSelectCte) - .build(); + .cteName( + selectContext.getNameGenerator().cteStepName(DistinctSelectCteStep.STRING_AGG, alias)) + .selects( + selects) + .fromTable(QueryStep.toTableLike(distinctSelectCte.getCteName())) + .groupBy( + ids.toFields()) + .predecessor(distinctSelectCte) + .build(); } private static QueryStep createDistinctSelectCte( - SingleColumnSqlSelect preprocessingSelect, - String alias, - SelectContext selectContext + SingleColumnSqlSelect preprocessingSelect, + String alias, + SelectContext selectContext ) { // values to aggregate must be event-filtered first String preprocessingTable = selectContext.getTables().cteName(ConceptCteStep.PREPROCESSING); @@ -124,16 +129,15 @@ private static QueryStep createDistinctSelectCte( SqlIdColumns ids = selectContext.getIds().qualify(preprocessingTable); - Selects selects = Selects.builder() - .ids(ids) - .sqlSelect(qualified) - .build(); + Selects selects = Selects.builder().ids(ids).sqlSelect(qualified).build(); return QueryStep.builder() - .cteName(selectContext.getNameGenerator().cteStepName(DistinctSelectCteStep.DISTINCT_SELECT, alias)) - .selectDistinct(true) - .selects(selects) - .fromTable(QueryStep.toTableLike(preprocessingTable)) - .build(); + .cteName( + selectContext.getNameGenerator().cteStepName(DistinctSelectCteStep.DISTINCT_SELECT, alias)) + .selectDistinct( + true) + .selects(selects) + .fromTable(QueryStep.toTableLike(preprocessingTable)) + .build(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java index 73cd607ae02..263be12a3b6 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java @@ -11,38 +11,41 @@ public class EventDateUnionSelectConverter implements SelectConverter { @Override - public ConnectorSqlSelects connectorSelect(EventDateUnionSelect select, SelectContext selectContext) { + public ConnectorSqlSelects connectorSelect( + EventDateUnionSelect select, + SelectContext selectContext) { FieldWrapper stringAggregation = createEventDateUnionAggregation(select, selectContext); - ExtractingSqlSelect finalSelect = stringAggregation.qualify(selectContext.getTables().getPredecessor(ConceptCteStep.AGGREGATION_FILTER)); + ExtractingSqlSelect finalSelect = stringAggregation.qualify( + selectContext.getTables().getPredecessor(ConceptCteStep.AGGREGATION_FILTER)); - return ConnectorSqlSelects.builder() - .eventDateSelect(stringAggregation) - .finalSelect(finalSelect) - .build(); + return ConnectorSqlSelects.builder().eventDateSelect(stringAggregation).finalSelect(finalSelect).build(); } @Override public ConceptSqlSelects conceptSelect(EventDateUnionSelect select, SelectContext selectContext) { FieldWrapper stringAggregation = createEventDateUnionAggregation(select, selectContext); - ExtractingSqlSelect finalSelect = stringAggregation.qualify(selectContext.getTables().getPredecessor(ConceptCteStep.UNIVERSAL_SELECTS)); + ExtractingSqlSelect finalSelect = stringAggregation.qualify( + selectContext.getTables().getPredecessor(ConceptCteStep.UNIVERSAL_SELECTS)); - return ConceptSqlSelects.builder() - .eventDateSelect(stringAggregation) - .finalSelect(finalSelect) - .build(); + return ConceptSqlSelects.builder().eventDateSelect(stringAggregation).finalSelect(finalSelect).build(); } - private static FieldWrapper createEventDateUnionAggregation(EventDateUnionSelect select, SelectContext selectContext) { + private static FieldWrapper createEventDateUnionAggregation( + EventDateUnionSelect select, + SelectContext selectContext) { - Preconditions.checkArgument(selectContext.getValidityDate().isPresent(), "Can't convert an EventDateUnionSelect without a validity date being present"); + Preconditions.checkArgument( + selectContext.getValidityDate().isPresent(), + "Can't convert an EventDateUnionSelect without a validity date being present"); ColumnDateRange validityDate = selectContext.getValidityDate().get(); SqlFunctionProvider functionProvider = selectContext.getFunctionProvider(); String alias = selectContext.getNameGenerator().selectName(select); - ColumnDateRange qualified = validityDate.qualify(selectContext.getTables().getPredecessor(ConceptCteStep.INTERVAL_PACKING_SELECTS)); + ColumnDateRange qualified = validityDate.qualify( + selectContext.getTables().getPredecessor(ConceptCteStep.INTERVAL_PACKING_SELECTS)); return new FieldWrapper<>(functionProvider.dateRangeAggregation(qualified).as(alias)); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java index 7a9b21307b3..461017362ba 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java @@ -18,32 +18,36 @@ public class EventDurationSumSelectConverter implements SelectConverter { @Override - public ConnectorSqlSelects connectorSelect(EventDurationSumSelect select, SelectContext selectContext) { + public ConnectorSqlSelects connectorSelect( + EventDurationSumSelect select, + SelectContext selectContext) { FieldWrapper stringAggregation = createEventDurationSumAggregation(select, selectContext); - ExtractingSqlSelect finalSelect = stringAggregation.qualify(selectContext.getTables().getPredecessor(ConceptCteStep.AGGREGATION_FILTER)); + ExtractingSqlSelect finalSelect = stringAggregation.qualify( + selectContext.getTables().getPredecessor(ConceptCteStep.AGGREGATION_FILTER)); - return ConnectorSqlSelects.builder() - .eventDateSelect(stringAggregation) - .finalSelect(finalSelect) - .build(); + return ConnectorSqlSelects.builder().eventDateSelect(stringAggregation).finalSelect(finalSelect).build(); } @Override - public ConceptSqlSelects conceptSelect(EventDurationSumSelect select, SelectContext selectContext) { + public ConceptSqlSelects conceptSelect( + EventDurationSumSelect select, + SelectContext selectContext) { FieldWrapper stringAggregation = createEventDurationSumAggregation(select, selectContext); - ExtractingSqlSelect finalSelect = stringAggregation.qualify(selectContext.getTables().getPredecessor(ConceptCteStep.UNIVERSAL_SELECTS)); + ExtractingSqlSelect finalSelect = stringAggregation.qualify( + selectContext.getTables().getPredecessor(ConceptCteStep.UNIVERSAL_SELECTS)); - return ConceptSqlSelects.builder() - .eventDateSelect(stringAggregation) - .finalSelect(finalSelect) - .build(); + return ConceptSqlSelects.builder().eventDateSelect(stringAggregation).finalSelect(finalSelect).build(); } - private FieldWrapper createEventDurationSumAggregation(EventDurationSumSelect select, SelectContext selectContext) { + private FieldWrapper createEventDurationSumAggregation( + EventDurationSumSelect select, + SelectContext selectContext) { - Preconditions.checkArgument(selectContext.getValidityDate().isPresent(), "Can't convert an EventDateUnionSelect without a validity date being present"); + Preconditions.checkArgument( + selectContext.getValidityDate().isPresent(), + "Can't convert an EventDateUnionSelect without a validity date being present"); String predecessorCteName = selectContext.getTables().getPredecessor(ConceptCteStep.INTERVAL_PACKING_SELECTS); ColumnDateRange qualified = selectContext.getValidityDate().get().qualify(predecessorCteName); ColumnDateRange asDualColumn = selectContext.getFunctionProvider().toDualColumn(qualified); @@ -52,10 +56,10 @@ private FieldWrapper createEventDurationSumAggregation(EventDuration String alias = selectContext.getNameGenerator().selectName(select); Field durationSum = DSL.sum( - DSL.when(containsInfinityDate(asDualColumn, functionProvider), DSL.inline(null, Integer.class)) - .otherwise(functionProvider.dateDistance(ChronoUnit.DAYS, asDualColumn.getStart(), asDualColumn.getEnd())) - ) - .as(alias); + DSL.when(containsInfinityDate(asDualColumn, functionProvider), DSL.inline(null, Integer.class)) + .otherwise( + functionProvider.dateDistance(ChronoUnit.DAYS, asDualColumn.getStart(), asDualColumn.getEnd())) + ).as(alias); return new FieldWrapper<>(durationSum); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java index f79444f66ed..f13215837ed 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java @@ -9,17 +9,13 @@ public class ExistsSelectConverter implements SelectConverter { @Override public ConnectorSqlSelects connectorSelect(ExistsSelect select, SelectContext selectContext) { ExistsSqlSelect existsSqlSelect = createExistsSelect(select, selectContext); - return ConnectorSqlSelects.builder() - .finalSelect(existsSqlSelect) - .build(); + return ConnectorSqlSelects.builder().finalSelect(existsSqlSelect).build(); } @Override public ConceptSqlSelects conceptSelect(ExistsSelect select, SelectContext selectContext) { ExistsSqlSelect existsSqlSelect = createExistsSelect(select, selectContext); - return ConceptSqlSelects.builder() - .finalSelect(existsSqlSelect) - .build(); + return ConceptSqlSelects.builder().finalSelect(existsSqlSelect).build(); } private static ExistsSqlSelect createExistsSelect(ExistsSelect select, SelectContext selectContext) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSqlSelect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSqlSelect.java index b6a15eaab50..4cfd855d7c4 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSqlSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSqlSelect.java @@ -1,17 +1,17 @@ package com.bakdata.conquery.sql.conversion.model.select; +import static org.jooq.impl.DSL.*; +import static org.jooq.impl.DSL.field; + +import java.util.Collections; +import java.util.List; + import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import org.jooq.Field; import org.jooq.Name; -import java.util.Collections; -import java.util.List; - -import static org.jooq.impl.DSL.*; -import static org.jooq.impl.DSL.field; - @RequiredArgsConstructor(access = AccessLevel.PRIVATE) public class ExistsSqlSelect implements SingleColumnSqlSelect { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExtractingSqlSelect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExtractingSqlSelect.java index 620c84efdaa..5d59def7c40 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExtractingSqlSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExtractingSqlSelect.java @@ -39,9 +39,9 @@ public List requiredColumns() { public ExtractingSqlSelect qualify(String qualifier) { Field aliased = aliased(); return new ExtractingSqlSelect<>( - qualifier, - aliased.getName(), - aliased.getType() + qualifier, + aliased.getName(), + aliased.getType() ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FieldWrapper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FieldWrapper.java index 42a877a2e0f..64f8ebadf8c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FieldWrapper.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FieldWrapper.java @@ -53,9 +53,9 @@ public List requiredColumns() { public ExtractingSqlSelect qualify(String qualifier) { Field aliased = aliased(); return new ExtractingSqlSelect<>( - qualifier, - aliased.getName(), - aliased.getType() + qualifier, + aliased.getName(), + aliased.getType() ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FirstValueSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FirstValueSelectConverter.java index f4e29c54319..b6271628223 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FirstValueSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FirstValueSelectConverter.java @@ -7,13 +7,15 @@ public class FirstValueSelectConverter implements SelectConverter { @Override - public ConnectorSqlSelects connectorSelect(FirstValueSelect select, SelectContext selectContext) { + public ConnectorSqlSelects connectorSelect( + FirstValueSelect select, + SelectContext selectContext) { return ValueSelectUtil.createValueSelect( - select.getColumn().resolve(), - selectContext.getNameGenerator().selectName(select), - Field::asc, - select.getSubstringRange(), - selectContext + select.getColumn().resolve(), + selectContext.getNameGenerator().selectName(select), + Field::asc, + select.getSubstringRange(), + selectContext ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/LastValueSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/LastValueSelectConverter.java index a98a8aa9e40..1494e77ca58 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/LastValueSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/LastValueSelectConverter.java @@ -7,11 +7,15 @@ public class LastValueSelectConverter implements SelectConverter { @Override - public ConnectorSqlSelects connectorSelect(LastValueSelect select, SelectContext selectContext) { + public ConnectorSqlSelects connectorSelect( + LastValueSelect select, + SelectContext selectContext) { return ValueSelectUtil.createValueSelect( - select.getColumn().resolve(), - selectContext.getNameGenerator().selectName(select), - Field::desc, select.getSubstringRange(), selectContext + select.getColumn().resolve(), + selectContext.getNameGenerator().selectName(select), + Field::desc, + select.getSubstringRange(), + selectContext ); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java index 100c74327de..13ad48bcf16 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java @@ -8,7 +8,9 @@ public class RandomValueSelectConverter implements SelectConverter { @Override - public ConnectorSqlSelects connectorSelect(RandomValueSelect select, SelectContext selectContext) { + public ConnectorSqlSelects connectorSelect( + RandomValueSelect select, + SelectContext selectContext) { ConnectorSqlTables tables = selectContext.getTables(); @@ -17,16 +19,19 @@ public ConnectorSqlSelects connectorSelect(RandomValueSelect select, SelectConte ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(rootTableName, columnName, Object.class); String alias = selectContext.getNameGenerator().selectName(select); - Field qualifiedRootSelect = rootSelect.qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_SELECT)).select(); + Field qualifiedRootSelect = rootSelect.qualify( + tables.getPredecessor(ConceptCteStep.AGGREGATION_SELECT)).select(); Field firstAggregation = selectContext.getFunctionProvider().random(qualifiedRootSelect).as(alias); FieldWrapper firstAggregationSqlSelect = new FieldWrapper<>(firstAggregation, columnName); - ExtractingSqlSelect finalSelect = firstAggregationSqlSelect.qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)); + ExtractingSqlSelect finalSelect = firstAggregationSqlSelect.qualify( + tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)); return ConnectorSqlSelects.builder() - .preprocessingSelect(rootSelect) - .aggregationSelect(firstAggregationSqlSelect) - .finalSelect(finalSelect) - .build(); + .preprocessingSelect(rootSelect) + .aggregationSelect( + firstAggregationSqlSelect) + .finalSelect(finalSelect) + .build(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java index 1239ef8d876..fd71a857927 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java @@ -30,19 +30,19 @@ public DialectBundle getDialectBundle() { } public static SelectContext create( - SqlIdColumns ids, - Optional validityDate, - ConnectorSqlTables tables, - ConversionContext conversionContext + SqlIdColumns ids, + Optional validityDate, + ConnectorSqlTables tables, + ConversionContext conversionContext ) { return new SelectContext<>(ids, validityDate, tables, conversionContext); } public static SelectContext create( - SqlIdColumns ids, - Optional validityDate, - ConceptSqlTables tables, - ConversionContext conversionContext + SqlIdColumns ids, + Optional validityDate, + ConceptSqlTables tables, + ConversionContext conversionContext ) { return new SelectContext<>(ids, validityDate, tables, conversionContext); } @@ -51,9 +51,9 @@ public SqlFunctionProvider getFunctionProvider() { return getDialectBundle().getFunctionProvider(); } - public SqlIdColumns getIds() { - return this.conversionContext.isWithStratification() - ? this.conversionContext.getStratificationTable().getSelects().getIds() - : this.ids; - } + public SqlIdColumns getIds() { + return this.conversionContext.isWithStratification() ? this.conversionContext.getStratificationTable() + .getSelects() + .getIds() : this.ids; + } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectConverter.java index 3e8a8bfb291..1b5c9f57c32 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectConverter.java @@ -7,11 +7,13 @@ public interface SelectConverter { default ConnectorSqlSelects connectorSelect(S select, SelectContext selectContext) { - throw new UnsupportedOperationException("Conversion of Select %s not implemented on Connector-level".formatted(select.getClass())); + throw new UnsupportedOperationException( + "Conversion of Select %s not implemented on Connector-level".formatted(select.getClass())); } default ConceptSqlSelects conceptSelect(S select, SelectContext selectContext) { - throw new UnsupportedOperationException("Conversion of Select %s not implemented or not possible on Concept-level".formatted(select.getClass())); + throw new UnsupportedOperationException( + "Conversion of Select %s not implemented or not possible on Concept-level".formatted(select.getClass())); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelect.java index 5ca54b46c41..a7a265b5cd0 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelect.java @@ -1,12 +1,12 @@ package com.bakdata.conquery.sql.conversion.model.select; +import java.util.List; + import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.ExistsSelect; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.Qualifiable; import org.jooq.Field; -import java.util.List; - public interface SqlSelect extends Qualifiable { List> toFields(); @@ -42,9 +42,11 @@ default SqlSelect toFinalRepresentation() { * Aggregate this select to one value per ID group in the final concept query. */ default List> aggregateForFinalQuery(SqlFunctionProvider functionProvider) { - return toFinalRepresentation().toFields().stream() - .>map(field -> functionProvider.anyValue(field).as(field.getName())) - .toList(); + return toFinalRepresentation().toFields() + .stream() + .>map( + field -> functionProvider.anyValue(field).as(field.getName())) + .toList(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java index e37d70b2a85..f72fb04a44b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java @@ -28,37 +28,45 @@ class ValueSelectUtil { public static ConnectorSqlSelects createValueSelect( - Column column, - String alias, - Function, ? extends SortField> ordering, - Range.IntegerRange substringRange, SelectContext selectContext) { + Column column, + String alias, + Function, ? extends SortField> ordering, + Range.IntegerRange substringRange, + SelectContext selectContext) { - SingleColumnSqlSelect rootSelect = MappableSingleColumnSelect.getSubstringSelect(column, substringRange, selectContext, alias); + SingleColumnSqlSelect rootSelect = MappableSingleColumnSelect.getSubstringSelect( + column, + substringRange, + selectContext, + alias); // create a CTE, that per row makes a window calculation to select for the rank of the validity date. // Further down below, we select the values with rank=1, which is FIRST/LAST depending on sort order supplied by the creator. - QueryStep rowNumberStep = - buildRowNumberStep(rootSelect, ordering, alias, selectContext); + QueryStep rowNumberStep = buildRowNumberStep(rootSelect, ordering, alias, selectContext); QueryStep rowFilterStep = buildRowFilterStep(rowNumberStep, alias, selectContext.getIds()); SqlSelect finalSelect = rowFilterStep.getQualifiedSelects().getSqlSelects().getFirst(); - FieldWrapper aggregationSelect = - new FieldWrapper<>(field(coalesce(finalSelect.qualify(ValueSelectCteStep.ROW_SELECT_STEP.cteName(alias)))).as(alias), column.getName()); + FieldWrapper aggregationSelect = new FieldWrapper<>( + field(coalesce(finalSelect.qualify(ValueSelectCteStep.ROW_SELECT_STEP.cteName(alias)))).as(alias), + column.getName()); return ConnectorSqlSelects.builder() - .additionalPredecessor(Optional.of(rowFilterStep)) - .preprocessingSelect(rootSelect) - .finalSelect(aggregationSelect) - .build(); + .additionalPredecessor(Optional.of(rowFilterStep)) + .preprocessingSelect( + rootSelect) + .finalSelect(aggregationSelect) + .build(); } private static QueryStep buildRowNumberStep( - SingleColumnSqlSelect rootSelect, Function, ? extends SortField> ordering, String alias, - SelectContext selectContext) { + SingleColumnSqlSelect rootSelect, + Function, ? extends SortField> ordering, + String alias, + SelectContext selectContext) { SqlFunctionProvider functionProvider = selectContext.getFunctionProvider(); @@ -66,65 +74,78 @@ private static QueryStep buildRowNumberStep( Field qualifiedRootSelect = rootSelect.qualify(predecessor).select(); return QueryStep.builder() - .selects(Selects.builder() - .ids(selectContext.getIds().qualify(null)) - .sqlSelects(List.of( - new FieldWrapper<>(qualifiedRootSelect.as(alias), qualifiedRootSelect.getName()), - rowNumberField(predecessor, selectContext.getValidityDate(), ordering, - selectContext.getIds(), - functionProvider - ) - )) - .build()) - .cteName(ValueSelectCteStep.ROW_NUMBER_STEP.cteName(alias)) - .conditions(List.of(qualifiedRootSelect.isNotNull())) - .fromTable(table(name(predecessor))) - .build(); + .selects( + Selects.builder() + .ids(selectContext.getIds().qualify(null)) + .sqlSelects( + List.of( + new FieldWrapper<>(qualifiedRootSelect.as(alias), qualifiedRootSelect.getName()), + rowNumberField( + predecessor, + selectContext.getValidityDate(), + ordering, + selectContext.getIds(), + functionProvider + ) + )) + .build()) + .cteName(ValueSelectCteStep.ROW_NUMBER_STEP.cteName(alias)) + .conditions( + List.of(qualifiedRootSelect.isNotNull())) + .fromTable(table(name(predecessor))) + .build(); } private static QueryStep buildRowFilterStep(QueryStep rowNumberStep, String alias, SqlIdColumns idColumns) { Field rowNumber = field(name("row-number")); SqlIdColumns unqualifiedIds = idColumns.qualify(null); - SelectConditionStep selectFirst = - select(unqualifiedIds.toFields()) - .select(field(name(alias)), field(name("select-result", "row-number"))) - .from(table(name(ValueSelectCteStep.ROW_NUMBER_STEP.cteName(alias))) - .as("select-result")) - .where(or(rowNumber.equal(inline(1)), rowNumber.isNull())); + SelectConditionStep selectFirst = select(unqualifiedIds.toFields()).select( + field(name(alias)), + field(name("select-result", "row-number"))) + .from( + table(name(ValueSelectCteStep.ROW_NUMBER_STEP.cteName(alias))).as("select-result")) + .where( + or(rowNumber.equal(inline(1)), rowNumber.isNull())); return QueryStep.builder() - .predecessor(rowNumberStep) - .selects(Selects.builder() - .ids(unqualifiedIds) - .sqlSelect(new FieldWrapper<>(selectFirst.field(alias))) - .build()) - .cteName(ValueSelectCteStep.ROW_SELECT_STEP.cteName(alias)) - .fromTable(selectFirst) - .build(); + .predecessor(rowNumberStep) + .selects( + Selects.builder() + .ids(unqualifiedIds) + .sqlSelect( + new FieldWrapper<>(selectFirst.field(alias))) + .build()) + .cteName( + ValueSelectCteStep.ROW_SELECT_STEP.cteName(alias)) + .fromTable(selectFirst) + .build(); } @NotNull private static FieldWrapper rowNumberField( - String predecessor, Optional validityDate, Function, ? extends SortField> ordering, - SqlIdColumns idColumns, SqlFunctionProvider functionProvider) { + String predecessor, + Optional validityDate, + Function, ? extends SortField> ordering, + SqlIdColumns idColumns, + SqlFunctionProvider functionProvider) { List> qualifiedIds = idColumns.qualify(predecessor).toFields(); if (validityDate.isEmpty()) { return new FieldWrapper<>( - rowNumber().over(partitionBy(qualifiedIds).orderBy(qualifiedIds)).as("row-number"), - new String[0] + rowNumber().over(partitionBy(qualifiedIds).orderBy(qualifiedIds)).as("row-number"), + new String[0] ); } return new FieldWrapper<>( - rowNumber().over( - partitionBy(qualifiedIds) - .orderBy(functionProvider.orderByValidityDates(ordering, validityDate.get().qualify(predecessor).toFields())) - ).as("row-number"), - new String[0] + rowNumber().over( + partitionBy(qualifiedIds).orderBy( + functionProvider.orderByValidityDates(ordering, validityDate.get().qualify(predecessor).toFields())) + ).as("row-number"), + new String[0] ); } @@ -132,7 +153,9 @@ private static FieldWrapper rowNumberField( @RequiredArgsConstructor @Getter enum ValueSelectCteStep implements CteStep { - ROW_NUMBER_STEP("value_select_assign_row_number_step", ConceptCteStep.PREPROCESSING), ROW_SELECT_STEP("value_select_first_row_step", ROW_NUMBER_STEP); + ROW_NUMBER_STEP("value_select_assign_row_number_step", ConceptCteStep.PREPROCESSING), ROW_SELECT_STEP( + "value_select_first_row_step", + ROW_NUMBER_STEP); private final String suffix; private final CteStep predecessor; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java index e63356c199e..6ca1b0edb4c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java @@ -34,14 +34,16 @@ public ConversionContext convert(AbsoluteFormQuery form, ConversionContext conte QueryStep convertedPrerequisite = convertPrerequisite(form, context); StratificationTableFactory tableFactory = new StratificationTableFactory(convertedPrerequisite, context); - QueryStep stratificationTable = tableFactory.createAbsoluteStratificationTable(form.getResolutionsAndAlignmentMap(), context); + QueryStep stratificationTable = tableFactory.createAbsoluteStratificationTable( + form.getResolutionsAndAlignmentMap(), + context); return formHelper.convertForm( - FormType.ABSOLUTE, - stratificationTable, - form.getFeatures(), - form.getResultInfos(), - context + FormType.ABSOLUTE, + stratificationTable, + form.getFeatures(), + form.getResultInfos(), + context ); } @@ -51,28 +53,38 @@ public ConversionContext convert(AbsoluteFormQuery form, ConversionContext conte */ private static QueryStep convertPrerequisite(AbsoluteFormQuery absoluteForm, ConversionContext context) { - ConversionContext withConvertedPrerequisite = context.getNodeConversions().convert(absoluteForm.getQuery(), context); - Preconditions.checkArgument(withConvertedPrerequisite.getQuerySteps().size() == 1, "Base query conversion should produce exactly 1 QueryStep"); + ConversionContext withConvertedPrerequisite = context.getNodeConversions() + .convert( + absoluteForm.getQuery(), + context); + Preconditions.checkArgument( + withConvertedPrerequisite.getQuerySteps().size() == 1, + "Base query conversion should produce exactly 1 QueryStep"); QueryStep convertedPrerequisite = withConvertedPrerequisite.getLastConvertedStep(); ColumnDateRange bounds = context.getDialectBundle() - .getFunctionProvider() - .forCDateRange(CDateRange.of(absoluteForm.getDateRange())).as(SharedAliases.STRATIFICATION_BOUNDS.getAlias()); + .getFunctionProvider() + .forCDateRange( + CDateRange.of(absoluteForm.getDateRange())) + .as(SharedAliases.STRATIFICATION_BOUNDS.getAlias()); Selects prerequisiteSelects = convertedPrerequisite.getQualifiedSelects(); // we only keep the primary column for the upcoming form Selects selects = Selects.builder() - .ids(new SqlIdColumns(prerequisiteSelects.getIds().getPrimaryColumn())) - .stratificationDate(Optional.of(bounds)) - .build(); + .ids( + new SqlIdColumns(prerequisiteSelects.getIds().getPrimaryColumn())) + .stratificationDate( + Optional.of(bounds)) + .build(); return QueryStep.builder() - .cteName(FormCteStep.EXTRACT_IDS.getSuffix()) - .selects(selects) - .fromTable(QueryStep.toTableLike(convertedPrerequisite.getCteName())) - .groupBy(selects.getIds().toFields()) // group by primary column to ensure max. 1 entry per subject - .predecessors(List.of(convertedPrerequisite)) - .build(); + .cteName(FormCteStep.EXTRACT_IDS.getSuffix()) + .selects(selects) + .fromTable( + QueryStep.toTableLike(convertedPrerequisite.getCteName())) + .groupBy(selects.getIds().toFields()) // group by primary column to ensure max. 1 entry per subject + .predecessors(List.of(convertedPrerequisite)) + .build(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java index 343b70a2ae9..43e21671ae7 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java @@ -1,5 +1,9 @@ package com.bakdata.conquery.sql.conversion.query; +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.concept.specific.CQNegation; import com.bakdata.conquery.models.query.DateAggregationMode; @@ -13,10 +17,6 @@ import org.jooq.*; import org.jooq.Record; -import java.util.List; -import java.util.Optional; -import java.util.stream.Stream; - @RequiredArgsConstructor public class ConceptQueryConverter implements NodeConverter { @@ -30,8 +30,8 @@ private static Selects getPreFinalSelects(QueryStep preFinalStep, ConversionCont } // adding extra selects List concatenated = Stream.concat( - preFinalStepSelects.getSqlSelects().stream(), - externalExtras.getQualifiedSelects().getSqlSelects().stream() + preFinalStepSelects.getSqlSelects().stream(), + externalExtras.getQualifiedSelects().getSqlSelects().stream() ).toList(); return preFinalStepSelects.toBuilder().sqlSelects(concatenated).build(); } @@ -42,9 +42,9 @@ private static TableLike getFinalTable(QueryStep preFinalStep, Conversio return QueryStep.toTableLike(preFinalStep.getCteName()); } return QueryStepJoiner.constructJoinedTable( - List.of(preFinalStep, externalExtras), - ConqueryJoinType.INNER_JOIN, - context + List.of(preFinalStep, externalExtras), + ConqueryJoinType.INNER_JOIN, + context ); } @@ -57,30 +57,43 @@ public Class getConversionClass() { public ConversionContext convert(ConceptQuery conceptQuery, ConversionContext context) { SqlFunctionProvider functionProvider = context.getDialectBundle().getFunctionProvider(); - ConversionContext contextAfterConversion = context.getNodeConversions().convert(conceptQuery.getRoot(), context); + ConversionContext contextAfterConversion = context.getNodeConversions() + .convert( + conceptQuery.getRoot(), + context); QueryStep preFinalStep = contextAfterConversion.getLastConvertedStep(); // negation of a single node results in an anti-join with all ids table if (preFinalStep.isNegate()) { - preFinalStep = QueryStepJoiner.antiJoinWithAllIdsTable(preFinalStep, contextAfterConversion, CQNegation.determineDateAction(conceptQuery.getDateAggregationMode())); + preFinalStep = QueryStepJoiner.antiJoinWithAllIdsTable( + preFinalStep, + contextAfterConversion, + CQNegation.determineDateAction(conceptQuery.getDateAggregationMode())); } Selects preFinalSelects = getPreFinalSelects(preFinalStep, contextAfterConversion); - List predecessors = Stream.concat(Stream.of(preFinalStep), Stream.ofNullable(contextAfterConversion.getExternalExtras())).toList(); + List predecessors = Stream.concat( + Stream.of(preFinalStep), + Stream.ofNullable(contextAfterConversion.getExternalExtras())).toList(); QueryStep finalStep = QueryStep.builder() - .cteName(null) // the final QueryStep won't be converted to a CTE - .selects(getFinalSelects(conceptQuery, preFinalSelects, functionProvider)) - .fromTable(getFinalTable(preFinalStep, contextAfterConversion)) - .groupBy(getFinalGroupBySelects(preFinalSelects)) - .predecessors(predecessors) - .build(); + .cteName(null) // the final QueryStep won't be converted to a CTE + .selects(getFinalSelects(conceptQuery, preFinalSelects, functionProvider)) + .fromTable( + getFinalTable(preFinalStep, contextAfterConversion)) + .groupBy( + getFinalGroupBySelects(preFinalSelects)) + .predecessors(predecessors) + .build(); Select finalQuery = this.queryStepTransformer.toSelectQuery(finalStep, functionProvider); return contextAfterConversion.withFinalQuery(new SqlQuery(finalQuery, conceptQuery.getResultInfos())); } - private Selects getFinalSelects(ConceptQuery conceptQuery, Selects preFinalSelects, SqlFunctionProvider functionProvider) { + private Selects getFinalSelects( + ConceptQuery conceptQuery, + Selects preFinalSelects, + SqlFunctionProvider functionProvider) { Selects finalSelects = preFinalSelects; if (conceptQuery.getDateAggregationMode() == DateAggregationMode.NONE) { finalSelects = preFinalSelects.blockValidityDate(); @@ -89,22 +102,30 @@ private Selects getFinalSelects(ConceptQuery conceptQuery, Selects preFinalSelec // In case all final selects have no validity-date, we convert it to infinity. if (preFinalSelects.getValidityDate().isEmpty()) { return preFinalSelects.toBuilder() - .validityDate(Optional.of(functionProvider.emptyColumnDateRange())) - .build(); + .validityDate( + Optional.of(functionProvider.emptyColumnDateRange())) + .build(); } return Selects.builder() - .ids(finalSelects.getIds()) - .validityDate(finalSelects.getValidityDate()) - .stratificationDate(finalSelects.getStratificationDate()) - .sqlSelects(getFinalAggregatedSelects(finalSelects, functionProvider)) - .build(); + .ids(finalSelects.getIds()) + .validityDate( + finalSelects.getValidityDate()) + .stratificationDate(finalSelects.getStratificationDate()) + .sqlSelects( + getFinalAggregatedSelects(finalSelects, functionProvider)) + .build(); } - private List> getFinalAggregatedSelects(Selects finalSelects, SqlFunctionProvider functionProvider) { - return finalSelects.getSqlSelects().stream() - .flatMap(sqlSelect -> sqlSelect.aggregateForFinalQuery(functionProvider).stream()) - .map(this::toFieldWrapper) - .toList(); + private List> getFinalAggregatedSelects( + Selects finalSelects, + SqlFunctionProvider functionProvider) { + return finalSelects.getSqlSelects() + .stream() + .flatMap( + sqlSelect -> sqlSelect.aggregateForFinalQuery(functionProvider).stream()) + .map( + this::toFieldWrapper) + .toList(); } private FieldWrapper toFieldWrapper(Field field) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java index 71bb59a32c2..a61e755355c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java @@ -32,66 +32,77 @@ public ConversionContext convert(EntityDateQuery entityDateQuery, ConversionCont QueryStep prerequisite = formHelper.convertPrerequisite(entityDateQuery.getQuery(), context); QueryStep withOverwrittenValidityDateBounds = overwriteBounds(prerequisite, entityDateQuery, context); - StratificationTableFactory tableFactory = new StratificationTableFactory(withOverwrittenValidityDateBounds, context); - QueryStep stratificationTable = tableFactory.createAbsoluteStratificationTable(entityDateQuery.getResolutionsAndAlignments(), context); + StratificationTableFactory tableFactory = new StratificationTableFactory( + withOverwrittenValidityDateBounds, + context); + QueryStep stratificationTable = tableFactory.createAbsoluteStratificationTable( + entityDateQuery.getResolutionsAndAlignments(), + context); return formHelper.convertForm( - FormType.ENTITY_DATE, - stratificationTable, - entityDateQuery.getFeatures(), - entityDateQuery.getResultInfos(), - context + FormType.ENTITY_DATE, + stratificationTable, + entityDateQuery.getFeatures(), + entityDateQuery.getResultInfos(), + context ); } /** * Computes the intersection of the entity date and the entity date query's daterange. */ - private static QueryStep overwriteBounds(QueryStep prerequisite, EntityDateQuery entityDateQuery, ConversionContext context) { + private static QueryStep overwriteBounds( + QueryStep prerequisite, + EntityDateQuery entityDateQuery, + ConversionContext context) { Preconditions.checkArgument( - prerequisite.getQualifiedSelects().getValidityDate().isPresent(), - "Expecting the prerequisite step's Selects to contain a validity date" + prerequisite.getQualifiedSelects().getValidityDate().isPresent(), + "Expecting the prerequisite step's Selects to contain a validity date" ); SqlFunctionProvider functionProvider = context.getFunctionProvider(); // we want to create a stratification for each distinct validity date range of an entity, // so we first need to unnest the validity date in case it is a multirange String unnestCteName = FormCteStep.UNNEST_ENTITY_DATE_CTE.getSuffix(); - QueryStep unnestedEntityDate = functionProvider.unnestDaterange(prerequisite.getSelects().getValidityDate().get(), prerequisite, unnestCteName); + QueryStep unnestedEntityDate = functionProvider.unnestDaterange( + prerequisite.getSelects().getValidityDate().get(), + prerequisite, + unnestCteName); Selects unnestedSelects = unnestedEntityDate.getQualifiedSelects(); ColumnDateRange withOverwrittenBounds; if (entityDateQuery.getDateRange() == null) { withOverwrittenBounds = unnestedSelects.getValidityDate().get(); - } - else { + } else { ColumnDateRange formDateRange = functionProvider.forCDateRange(entityDateQuery.getDateRange()); ColumnDateRange entityDate = unnestedSelects.getValidityDate().get(); withOverwrittenBounds = functionProvider.intersection(formDateRange, entityDate) - .as(SharedAliases.STRATIFICATION_BOUNDS.getAlias()); + .as( + SharedAliases.STRATIFICATION_BOUNDS.getAlias()); } Selects selects = Selects.builder() - .ids(unnestedSelects.getIds()) - .stratificationDate(Optional.of(withOverwrittenBounds)) - .build(); + .ids(unnestedSelects.getIds()) + .stratificationDate( + Optional.of(withOverwrittenBounds)) + .build(); List predecessors; // unnest step is optional depending on the dialect if (unnestedEntityDate == prerequisite) { predecessors = List.of(prerequisite); - } - else { + } else { predecessors = List.of(prerequisite, unnestedEntityDate); } return QueryStep.builder() - .cteName(FormCteStep.OVERWRITE_BOUNDS.getSuffix()) - .selects(selects) - .fromTable(QueryStep.toTableLike(unnestedEntityDate.getCteName())) - .predecessors(predecessors) - .build(); + .cteName(FormCteStep.OVERWRITE_BOUNDS.getSuffix()) + .selects(selects) + .fromTable( + QueryStep.toTableLike(unnestedEntityDate.getCteName())) + .predecessors(predecessors) + .build(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java index 75b32f027bf..4a687ae4e79 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java @@ -1,5 +1,12 @@ package com.bakdata.conquery.sql.conversion.query; +import static org.jooq.impl.DSL.*; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + import com.bakdata.conquery.apiv1.forms.FeatureGroup; import com.bakdata.conquery.apiv1.query.ArrayConceptQuery; import com.bakdata.conquery.apiv1.query.ConceptQuery; @@ -18,13 +25,6 @@ import org.jooq.*; import org.jooq.Record; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.jooq.impl.DSL.*; - @RequiredArgsConstructor public class FormConversionHelper { @@ -35,9 +35,9 @@ public class FormConversionHelper { * For {@link FormType#RELATIVE}, the {@link FeatureGroup} will be set too. */ private static Selects getFinalSelects( - FormType formType, - QueryStep stratificationTable, - QueryStep convertedFeatures + FormType formType, + QueryStep stratificationTable, + QueryStep convertedFeatures ) { Selects preFinalSelects = convertedFeatures.getQualifiedSelects(); Selects stratificationSelects = stratificationTable.getQualifiedSelects(); @@ -45,16 +45,21 @@ private static Selects getFinalSelects( SqlIdColumns ids = stratificationSelects.getIds().forFinalSelect(); Selects.SelectsBuilder selects = Selects.builder() - .ids(ids) - .validityDate(preFinalSelects.getValidityDate()) - .stratificationDate(stratificationSelects.getStratificationDate()); + .ids(ids) + .validityDate( + preFinalSelects.getValidityDate()) + .stratificationDate(stratificationSelects.getStratificationDate()); // relative forms have FeatureGroup information after the stratification date and before all other selects if (formType.equals(FormType.RELATIVE)) { - Field indexField = field(name(stratificationTable.getCteName(), SharedAliases.INDEX.getAlias()), Integer.class); - Field scope = when(indexField.isNull().or(indexField.lessThan(0)), inline(FeatureGroup.FEATURE.toString())) - .otherwise(inline(FeatureGroup.OUTCOME.toString())) - .as(SharedAliases.OBSERVATION_SCOPE.getAlias()); + Field indexField = field( + name(stratificationTable.getCteName(), SharedAliases.INDEX.getAlias()), + Integer.class); + Field scope = when( + indexField.isNull().or(indexField.lessThan(0)), + inline(FeatureGroup.FEATURE.toString())).otherwise(inline(FeatureGroup.OUTCOME.toString())) + .as( + SharedAliases.OBSERVATION_SCOPE.getAlias()); selects = selects.sqlSelect(new FieldWrapper<>(scope)); } @@ -69,22 +74,25 @@ private static Selects getFinalSelects( public QueryStep convertPrerequisite(Query query, ConversionContext context) { ConversionContext withConvertedPrerequisite = context.getNodeConversions().convert(query, context); - Preconditions.checkArgument(withConvertedPrerequisite.getQuerySteps().size() == 1, "Base query conversion should produce exactly 1 QueryStep"); + Preconditions.checkArgument( + withConvertedPrerequisite.getQuerySteps().size() == 1, + "Base query conversion should produce exactly 1 QueryStep"); QueryStep convertedPrerequisite = withConvertedPrerequisite.getLastConvertedStep(); Selects prerequisiteSelects = convertedPrerequisite.getQualifiedSelects(); // we keep the primary column and the validity date Selects selects = Selects.builder() - .ids(new SqlIdColumns(prerequisiteSelects.getIds().getPrimaryColumn())) - .validityDate(prerequisiteSelects.getValidityDate()) - .build(); + .ids( + new SqlIdColumns(prerequisiteSelects.getIds().getPrimaryColumn())) + .validityDate( + prerequisiteSelects.getValidityDate()) + .build(); // we want to keep each primary column and the corresponding distinct validity date ranges List> groupByFields = Stream.concat( - Stream.of(prerequisiteSelects.getIds().getPrimaryColumn()), - prerequisiteSelects.getValidityDate().stream().flatMap(validityDate -> validityDate.toFields().stream()) - ) - .collect(Collectors.toList()); + Stream.of(prerequisiteSelects.getIds().getPrimaryColumn()), + prerequisiteSelects.getValidityDate().stream().flatMap(validityDate -> validityDate.toFields().stream()) + ).collect(Collectors.toList()); // filter out entries with a null validity date Optional columnDateRange = prerequisiteSelects.getValidityDate(); @@ -97,21 +105,24 @@ public QueryStep convertPrerequisite(Query query, ConversionContext context) { } return QueryStep.builder() - .cteName(FormCteStep.EXTRACT_IDS.getSuffix()) - .selects(selects) - .fromTable(QueryStep.toTableLike(convertedPrerequisite.getCteName())) - .conditions(List.of(dateNotNullCondition)) - .groupBy(groupByFields) - .predecessors(List.of(convertedPrerequisite)) - .build(); + .cteName(FormCteStep.EXTRACT_IDS.getSuffix()) + .selects(selects) + .fromTable( + QueryStep.toTableLike(convertedPrerequisite.getCteName())) + .conditions( + List.of(dateNotNullCondition)) + .groupBy(groupByFields) + .predecessors( + List.of(convertedPrerequisite)) + .build(); } public ConversionContext convertForm( - FormType formType, - QueryStep stratificationTable, - ArrayConceptQuery features, - List resultInfos, - ConversionContext context + FormType formType, + QueryStep stratificationTable, + ArrayConceptQuery features, + List resultInfos, + ConversionContext context ) { // feature conversion ConversionContext childContext = context.createChildContext().withStratificationTable(stratificationTable); @@ -121,7 +132,11 @@ public ConversionContext convertForm( // child context contains the converted feature's QuerySteps List queriesToJoin = childContext.getQuerySteps(); - QueryStep joinedFeatures = QueryStepJoiner.joinSteps(queriesToJoin, ConqueryJoinType.OUTER_JOIN, DateAggregationAction.BLOCK, context); + QueryStep joinedFeatures = QueryStepJoiner.joinSteps( + queriesToJoin, + ConqueryJoinType.OUTER_JOIN, + DateAggregationAction.BLOCK, + context); return createFinalSelect(formType, stratificationTable, joinedFeatures, resultInfos, context); } @@ -133,27 +148,33 @@ public ConversionContext convertForm( * features with the full stratification table again. */ private ConversionContext createFinalSelect( - FormType formType, - QueryStep stratificationTable, - QueryStep convertedFeatures, - List resultInfos, - ConversionContext context + FormType formType, + QueryStep stratificationTable, + QueryStep convertedFeatures, + List resultInfos, + ConversionContext context ) { Preconditions.checkArgument( - stratificationTable.getSelects().getStratificationDate().isPresent() && convertedFeatures.getSelects().getStratificationDate().isPresent(), - "Expecting stratification table and converted features table to contain a stratification date" + stratificationTable.getSelects().getStratificationDate().isPresent() && convertedFeatures.getSelects() + .getStratificationDate() + .isPresent(), + "Expecting stratification table and converted features table to contain a stratification date" ); List queriesToJoin = List.of(stratificationTable, convertedFeatures); - TableLike joinedTable = QueryStepJoiner.constructJoinedTable(queriesToJoin, ConqueryJoinType.LEFT_JOIN, context); + TableLike joinedTable = QueryStepJoiner.constructJoinedTable( + queriesToJoin, + ConqueryJoinType.LEFT_JOIN, + context); SqlFunctionProvider functionProvider = context.getFunctionProvider(); QueryStep finalStep = QueryStep.builder() - .cteName(null) // the final QueryStep won't be converted to a CTE - .selects(getFinalSelects(formType, stratificationTable, convertedFeatures)) - .fromTable(joinedTable) - .predecessors(queriesToJoin) - .build(); + .cteName(null) // the final QueryStep won't be converted to a CTE + .selects(getFinalSelects(formType, stratificationTable, convertedFeatures)) + .fromTable( + joinedTable) + .predecessors(queriesToJoin) + .build(); Select selectQuery = queryStepTransformer.toSelectQuery(finalStep, functionProvider); return context.withFinalQuery(new SqlQuery(selectQuery, resultInfos)); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/RelativFormQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/RelativFormQueryConverter.java index e236e0d054f..cdb6b6f2b3c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/RelativFormQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/RelativFormQueryConverter.java @@ -26,11 +26,11 @@ public ConversionContext convert(RelativeFormQuery form, ConversionContext conte QueryStep stratificationTable = tableFactory.createRelativeStratificationTable(form, context); return formHelper.convertForm( - FormType.RELATIVE, - stratificationTable, - form.getFeatures(), - form.getResultInfos(), - context + FormType.RELATIVE, + stratificationTable, + form.getFeatures(), + form.getResultInfos(), + context ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/SecondaryIdQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/SecondaryIdQueryConverter.java index 8b59f9c70e8..b96ef583b12 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/SecondaryIdQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/SecondaryIdQueryConverter.java @@ -16,12 +16,15 @@ public Class getConversionClass() { @Override public ConversionContext convert(SecondaryIdQuery query, ConversionContext context) { - ConversionContext withConvertedQuery = context.getNodeConversions().convert( + ConversionContext withConvertedQuery = context.getNodeConversions() + .convert( query.getQuery(), context.withSecondaryIdDescription(query.getSecondaryId().resolve()) - ); + ); - Preconditions.checkArgument(withConvertedQuery.getFinalQuery() != null, "The SecondaryIdQuery's query should be converted by now."); + Preconditions.checkArgument( + withConvertedQuery.getFinalQuery() != null, + "The SecondaryIdQuery's query should be converted by now."); SqlQuery secondaryIdSqlQuery = withConvertedQuery.getFinalQuery().overwriteResultInfos(query.getResultInfos()); return withConvertedQuery.withFinalQuery(secondaryIdSqlQuery); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java index 5fe2b44e417..db9998ea393 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java @@ -1,5 +1,14 @@ package com.bakdata.conquery.sql.conversion.query; +import static org.jooq.impl.DSL.*; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.apiv1.query.TableExportQuery; import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; @@ -20,15 +29,6 @@ import org.jooq.*; import org.jooq.Record; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.jooq.impl.DSL.*; - @RequiredArgsConstructor public class TableExportQueryConverter implements NodeConverter { @@ -45,22 +45,29 @@ public class TableExportQueryConverter implements NodeConverter positions, - ConversionContext context + CQTable cqTable, + CQConcept concept, + CDateRange dateRestriction, + QueryStep convertedPrerequisite, + Map positions, + ConversionContext context ) { - final Field primaryColumn = TablePrimaryColumnUtil.findPrimaryColumn(cqTable.getConnector().resolve().getResolvedTable(), - context.getDefaultPrimaryColumn() + final Field primaryColumn = TablePrimaryColumnUtil.findPrimaryColumn( + cqTable.getConnector().resolve().getResolvedTable(), + context.getDefaultPrimaryColumn() ); final SqlIdColumns ids = new SqlIdColumns(primaryColumn); - final String conceptConnectorName = - context.getNameGenerator().conceptConnectorName(concept, cqTable.getConnector().resolve(), context.getSqlPrintSettings().getLocale()); - final Optional validityDate = convertTablesValidityDate(cqTable, conceptConnectorName, context); + final String conceptConnectorName = context.getNameGenerator() + .conceptConnectorName( + concept, + cqTable.getConnector().resolve(), + context.getSqlPrintSettings().getLocale()); + final Optional validityDate = convertTablesValidityDate( + cqTable, + conceptConnectorName, + context); final List> exportColumns = initializeFields(cqTable, positions); - final Selects selects = Selects.builder() - .ids(ids) - .validityDate(validityDate) - .sqlSelects(exportColumns) - .build(); + final Selects selects = Selects.builder().ids(ids).validityDate(validityDate).sqlSelects(exportColumns).build(); - final List filters = cqTable.getFilters().stream().map(filterValue -> filterValue.convertForTableExport(ids, context)).toList(); - final Table joinedTable = joinConnectorTableWithPrerequisite(cqTable, ids, convertedPrerequisite, dateRestriction, context); + final List filters = cqTable.getFilters() + .stream() + .map( + filterValue -> filterValue.convertForTableExport(ids, context)) + .toList(); + final Table joinedTable = joinConnectorTableWithPrerequisite( + cqTable, + ids, + convertedPrerequisite, + dateRestriction, + context); return QueryStep.builder() - .cteName(conceptConnectorName) - .selects(selects) - .fromTable(joinedTable) - .conditions(filters) - .forTableExport(true) - .build(); + .cteName(conceptConnectorName) + .selects(selects) + .fromTable(joinedTable) + .conditions( + filters) + .forTableExport(true) + .build(); } - private static Optional convertTablesValidityDate(CQTable table, String alias, ConversionContext context) { + private static Optional convertTablesValidityDate( + CQTable table, + String alias, + ConversionContext context) { final SqlFunctionProvider functionProvider = context.getFunctionProvider(); @@ -135,11 +158,11 @@ private static List> initializeFields(CQTable cqTable, Map joinConnectorTableWithPrerequisite( - CQTable cqTable, - SqlIdColumns ids, - QueryStep convertedPrerequisite, - CDateRange dateRestriction, - ConversionContext context + CQTable cqTable, + SqlIdColumns ids, + QueryStep convertedPrerequisite, + CDateRange dateRestriction, + ConversionContext context ) { final SqlFunctionProvider functionProvider = context.getFunctionProvider(); final Table connectorTable = table(name(cqTable.getConnector().resolve().resolveTableId().getTable())); @@ -147,8 +170,8 @@ private static Table joinConnectorTableWithPrerequisite( final ColumnDateRange validityDate = functionProvider.forValidityDate(cqTable.findValidityDate()); final List joinConditions = Stream.concat( - ids.join(convertedPrerequisite.getQualifiedSelects().getIds()).stream(), - Stream.of(functionProvider.dateRestriction(functionProvider.forCDateRange(dateRestriction), validityDate)) + ids.join(convertedPrerequisite.getQualifiedSelects().getIds()).stream(), + Stream.of(functionProvider.dateRestriction(functionProvider.forCDateRange(dateRestriction), validityDate)) ).toList(); return functionProvider.innerJoin(connectorTable, convertedPrerequisiteTable, joinConditions); @@ -175,8 +198,7 @@ private static Field createSourceInfoSelect(CQTable cqTable) { private static Field createColumnSelect(Column column, int position) { final String columnName = "%s-%s".formatted(column.getName(), position); - return field(name(column.getTable().getName(), column.getName())) - .as(columnName); + return field(name(column.getTable().getName(), column.getName())).as(columnName); } @Override @@ -191,24 +213,32 @@ public ConversionContext convert(TableExportQuery tableExportQuery, ConversionCo final Map positions = tableExportQuery.getPositions(); final CDateRange dateRestriction = CDateRange.of(tableExportQuery.getDateRange()); - final List convertedTables = tableExportQuery.getConcepts().stream() - .flatMap(concept -> concept.getTables().stream().map(table -> convertTable( - table, - concept, - dateRestriction, - convertedPrerequisite, - positions, - context - ))) - .toList(); + final List convertedTables = tableExportQuery.getConcepts() + .stream() + .flatMap( + concept -> concept.getTables() + .stream() + .map( + table -> convertTable( + table, + concept, + dateRestriction, + convertedPrerequisite, + positions, + context + ))) + .toList(); final QueryStep unionedTables = QueryStep.createUnionAllStep( - convertedTables, - null, // no CTE name required as this step will be the final select - List.of(convertedPrerequisite), - context.isNegation(), context.getFunctionProvider() + convertedTables, + null, // no CTE name required as this step will be the final select + List.of(convertedPrerequisite), + context.isNegation(), + context.getFunctionProvider() ); - final Select selectQuery = queryStepTransformer.toSelectQuery(unionedTables, context.getFunctionProvider()); + final Select selectQuery = queryStepTransformer.toSelectQuery( + unionedTables, + context.getFunctionProvider()); return context.withFinalQuery(new SqlQuery(selectQuery, tableExportQuery.getResultInfos())); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultCDateSetParser.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultCDateSetParser.java index a42d8273001..a30cdff2a0a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultCDateSetParser.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultCDateSetParser.java @@ -49,10 +49,14 @@ public List> toEpochDayRangeList(String multiDateRange) { StringReader reader = new StringReader(multiDateRange); Scanner scanner = new Scanner(reader) - // ) and ] are handled manually to determine real end-date - .useDelimiter(Pattern.compile("[%s]+".formatted(Stream.of(dateSetSep, dateRangeSep, dateSetStart, dateSetEnd, dateRangeBeginInclusive) - .map(Pattern::quote) - .collect(Collectors.joining())))); + // ) and ] are handled manually to determine real end-date + .useDelimiter( + Pattern.compile( + "[%s]+".formatted( + Stream.of(dateSetSep, dateRangeSep, dateSetStart, dateSetEnd, dateRangeBeginInclusive) + .map( + Pattern::quote) + .collect(Collectors.joining())))); ArrayList> out = new ArrayList<>(); @@ -87,8 +91,7 @@ private List tupleFromStrings(String begin, String end) { if (begin.equals(dateRangeMinValue)) { startDate = CDateRange.NEGATIVE_INFINITY; - } - else { + } else { startDate = CDate.ofLocalDate(dateReader.parseToLocalDate(begin)); } @@ -97,8 +100,7 @@ private List tupleFromStrings(String begin, String end) { if (endDateExpression.equals(dateRangeMaxValue)) { endDate = CDateRange.POSITIVE_INFINITY; - } - else { + } else { LocalDate dateValue = dateReader.parseToLocalDate(endDateExpression); endDate = CDate.ofLocalDate(dateValue); if (end.endsWith(dateRangeEndExclusive)) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java index 93ab63fa244..40b1ebc1247 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java @@ -24,7 +24,11 @@ public class DefaultResultSetProcessor implements ResultSetProcessor { *

* For example, calling a primitives' ResultSet getter like getDouble, getInt etc. straightaway will never return null. */ - private static T checkForNullElseGet(ResultSet resultSet, int columnIndex, Getter getter, Class resultType) throws SQLException { + private static T checkForNullElseGet( + ResultSet resultSet, + int columnIndex, + Getter getter, + Class resultType) throws SQLException { if (resultSet.getObject(columnIndex) == null) { return null; @@ -104,10 +108,9 @@ public List getDoubleList(ResultSet resultSet, int columnIndex) throws S @Override public List getMoneyList(ResultSet resultSet, int columnIndex) throws SQLException { return list( - resultSet, - columnIndex, - (string) -> BigDecimal.valueOf(Double.parseDouble(string)) - .setScale(2, RoundingMode.HALF_EVEN) + resultSet, + columnIndex, + (string) -> BigDecimal.valueOf(Double.parseDouble(string)).setScale(2, RoundingMode.HALF_EVEN) ); } @@ -120,16 +123,20 @@ private Number parseWithDateReader(String string) { return config.getLocale().getDateReader().parseToLocalDate(string).toEpochDay(); } - private List list(ResultSet resultSet, int columnIndex, Function parseFunction) throws SQLException { + private List list( + ResultSet resultSet, + int columnIndex, + Function parseFunction) throws SQLException { String arrayExpression = resultSet.getString(columnIndex); if (arrayExpression == null) { return null; } List result = Arrays.stream(arrayExpression.split(String.valueOf(UNIT_SEPARATOR))) - .filter(Strings::isNotBlank) - .map(parseFunction) - .toList(); + .filter( + Strings::isNotBlank) + .map(parseFunction) + .toList(); return result.isEmpty() ? null : result; } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/HanaSqlCDateSetParser.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/HanaSqlCDateSetParser.java index 81fe66563e3..60b1b71ff3d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/execution/HanaSqlCDateSetParser.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/HanaSqlCDateSetParser.java @@ -21,17 +21,17 @@ public class HanaSqlCDateSetParser extends DefaultCDateSetParser { public HanaSqlCDateSetParser() { super( - new DateReader(Set.of("yyyy-M-dd"), Collections.emptyList(), Collections.emptyList()), - Character.toString(DATE_SET_SEPARATOR), - DATE_RANGE_SEP, - HanaSqlFunctionProvider.MIN_DATE_VALUE, - HanaSqlFunctionProvider.MAX_DATE_VALUE, - DATE_RANGE_END_EXCLUSIVE, - DATE_RANGE_END_INCLUSIVE, - DATE_RANGE_BEGIN_INCLUSIVE, - DATE_SET_START, - DATE_SET_END, - EMPTY_DATE_SET + new DateReader(Set.of("yyyy-M-dd"), Collections.emptyList(), Collections.emptyList()), + Character.toString(DATE_SET_SEPARATOR), + DATE_RANGE_SEP, + HanaSqlFunctionProvider.MIN_DATE_VALUE, + HanaSqlFunctionProvider.MAX_DATE_VALUE, + DATE_RANGE_END_EXCLUSIVE, + DATE_RANGE_END_INCLUSIVE, + DATE_RANGE_BEGIN_INCLUSIVE, + DATE_SET_START, + DATE_SET_END, + EMPTY_DATE_SET ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessor.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessor.java index 6dbc880030d..54e6aa98d74 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessor.java @@ -65,7 +65,6 @@ static Reader readerForType(ResultType type, ResultSetProcessor processor) { @FunctionalInterface interface Reader { - T read(ResultSet resultSet, int columnIndex) - throws SQLException; + T read(ResultSet resultSet, int columnIndex) throws SQLException; } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlEntityResult.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlEntityResult.java index d98d7f9bb0e..86ae062b463 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlEntityResult.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlEntityResult.java @@ -16,8 +16,8 @@ @ToString @Getter @Setter -@AllArgsConstructor(onConstructor_=@JsonCreator) -@CPSType(id="SQL_RESULT", base= EntityResult.class) +@AllArgsConstructor(onConstructor_ = @JsonCreator) +@CPSType(id = "SQL_RESULT", base = EntityResult.class) public class SqlEntityResult implements EntityResult { private final String id; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionExecutionInfo.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionExecutionInfo.java index 1e3a5b00cb1..4399250eba8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionExecutionInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionExecutionInfo.java @@ -28,7 +28,12 @@ public SqlExecutionExecutionInfo() { this.executingLock = new CountDownLatch(1); } - public SqlExecutionExecutionInfo(ExecutionState executionState, List columnNames, List table, List resultInfos, CountDownLatch executingLock) { + public SqlExecutionExecutionInfo( + ExecutionState executionState, + List columnNames, + List table, + List resultInfos, + CountDownLatch executingLock) { this.executionState = executionState; this.columnNames = columnNames; this.resultInfos = resultInfos; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java index ec88a9a1632..3cff3e06af2 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java @@ -40,7 +40,8 @@ public class SqlExecutionService { public SqlExecutionExecutionInfo execute(SqlQuery sqlQuery) { - final SqlExecutionExecutionInfo result = dslContext.connectionResult(connection -> createStatementAndExecute(sqlQuery, connection)); + final SqlExecutionExecutionInfo result = dslContext.connectionResult( + connection -> createStatementAndExecute(sqlQuery, connection)); return result; } @@ -50,18 +51,24 @@ private SqlExecutionExecutionInfo createStatementAndExecute(SqlQuery sqlQuery, C final String sqlString = sqlQuery.getSql(); List resultInfos = sqlQuery.getResultInfos(); final List> resultTypes = resultInfos.stream() - .map(resultInfo -> resultInfo.createReader(resultSetProcessor)) - .collect(Collectors.toList()); + .map( + resultInfo -> resultInfo.createReader(resultSetProcessor)) + .collect(Collectors.toList()); log.info("Executing query: \n{}", sqlString); - try (Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery(sqlString)) { + try (Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery( + sqlString)) { final int columnCount = resultSet.getMetaData().getColumnCount(); final List columnNames = getColumnNames(resultSet, columnCount); final List resultTable = createResultTable(resultSet, resultTypes, columnCount); - return new SqlExecutionExecutionInfo(ExecutionState.RUNNING, columnNames, resultTable, resultInfos, new CountDownLatch(1)); + return new SqlExecutionExecutionInfo( + ExecutionState.RUNNING, + columnNames, + resultTable, + resultInfos, + new CountDownLatch(1)); } // not all DB vendors throw SQLExceptions catch (SQLException | RuntimeException e) { @@ -73,11 +80,15 @@ private SqlExecutionExecutionInfo createStatementAndExecute(SqlQuery sqlQuery, C private List getColumnNames(ResultSet resultSet, int columnCount) { // JDBC ResultSet indices start with 1 return IntStream.rangeClosed(1, columnCount) - .mapToObj(columnIndex -> getColumnName(resultSet, columnIndex)) - .toList(); + .mapToObj( + columnIndex -> getColumnName(resultSet, columnIndex)) + .toList(); } - private List createResultTable(ResultSet resultSet, List> resultTypes, int columnCount) throws SQLException { + private List createResultTable( + ResultSet resultSet, + List> resultTypes, + int columnCount) throws SQLException { final List resultTable = new ArrayList<>(resultSet.getFetchSize()); while (resultSet.next()) { final SqlEntityResult resultRow = getResultRow(resultSet, resultTypes, columnCount); @@ -89,13 +100,15 @@ private List createResultTable(ResultSet resultSet, List> resultTypes, int columnCount) throws SQLException { + private SqlEntityResult getResultRow( + ResultSet resultSet, + List> resultTypes, + int columnCount) throws SQLException { final String id = resultSet.getString(PID_COLUMN_INDEX); final Object[] resultRow = new Object[columnCount - 1]; @@ -121,8 +134,7 @@ public Stream fetchStream(Select query) { log.debug("Executing query: \n{}", query); try { return dslContext.fetchStream(query); - } - catch (DataAccessException exception) { + } catch (DataAccessException exception) { throw new ConqueryError.SqlError(exception); } } diff --git a/backend/src/main/java/com/bakdata/conquery/tasks/LoadStorageTask.java b/backend/src/main/java/com/bakdata/conquery/tasks/LoadStorageTask.java index 6496e9a1c64..30da3210889 100644 --- a/backend/src/main/java/com/bakdata/conquery/tasks/LoadStorageTask.java +++ b/backend/src/main/java/com/bakdata/conquery/tasks/LoadStorageTask.java @@ -41,8 +41,10 @@ public void execute(Map> parameters, PrintWriter output) th if (metaStorage != null) { storages.put("metaStorage", metaStorage); } - Map namespacedStorages = namespacedStorageProvider.getAllDatasetIds().stream() - .collect(Collectors.toMap(DatasetId::getName, namespacedStorageProvider::getStorage)); + Map namespacedStorages = namespacedStorageProvider.getAllDatasetIds() + .stream() + .collect( + Collectors.toMap(DatasetId::getName, namespacedStorageProvider::getStorage)); storages.putAll(namespacedStorages); @@ -53,16 +55,16 @@ public void execute(Map> parameters, PrintWriter output) th if (actions.contains(Action.LIST)) { // Just print out all the storages we have and return storages.keySet().forEach(output::println); - if (actions.size() > 1 ) { + if (actions.size() > 1) { log.debug("Encountered action {}. Ignoring other actions: {}", Action.LIST, actions); } return; } // Extract the requested storages - Map requestedStorages = parameters.getOrDefault("storage", Collections.emptyList()) - .stream() - .collect(Collectors.toMap(Function.identity(), storages::get)); + Map requestedStorages = parameters.getOrDefault( + "storage", + Collections.emptyList()).stream().collect(Collectors.toMap(Function.identity(), storages::get)); if (requestedStorages.isEmpty()) { // load all if request was empty @@ -109,8 +111,6 @@ private static void loadStorage(PrintWriter output, String storageName, Conquery } enum Action { - LOAD, - INVALIDATE_CACHE, - LIST + LOAD, INVALIDATE_CACHE, LIST } } diff --git a/backend/src/main/java/com/bakdata/conquery/tasks/PermissionCleanupTask.java b/backend/src/main/java/com/bakdata/conquery/tasks/PermissionCleanupTask.java index 7eb5ee171b9..fedf11855af 100644 --- a/backend/src/main/java/com/bakdata/conquery/tasks/PermissionCleanupTask.java +++ b/backend/src/main/java/com/bakdata/conquery/tasks/PermissionCleanupTask.java @@ -39,14 +39,30 @@ public PermissionCleanupTask(MetaStorage storage) { @Override public void execute(Map> parameters, PrintWriter output) throws Exception { - log.info("Permissions deleted from all users: {}", deleteQueryPermissionsWithMissingRef(storage, storage.getAllUsers())); - log.info("Permissions deleted from all groups: {}", deleteQueryPermissionsWithMissingRef(storage, storage.getAllGroups())); - log.info("Permissions deleted from all roles: {}", deleteQueryPermissionsWithMissingRef(storage, storage.getAllRoles())); - log.info("Owned Execution permissions deleted: {}", - deletePermissionsOfOwnedInstances(storage, ExecutionPermission.DOMAIN.toLowerCase(), ManagedExecutionId.Parser.INSTANCE, storage::getExecution) + log.info( + "Permissions deleted from all users: {}", + deleteQueryPermissionsWithMissingRef(storage, storage.getAllUsers())); + log.info( + "Permissions deleted from all groups: {}", + deleteQueryPermissionsWithMissingRef(storage, storage.getAllGroups())); + log.info( + "Permissions deleted from all roles: {}", + deleteQueryPermissionsWithMissingRef(storage, storage.getAllRoles())); + log.info( + "Owned Execution permissions deleted: {}", + deletePermissionsOfOwnedInstances( + storage, + ExecutionPermission.DOMAIN.toLowerCase(), + ManagedExecutionId.Parser.INSTANCE, + storage::getExecution) ); - log.info("Owned FormConfig permissions deleted: {}", - deletePermissionsOfOwnedInstances(storage, FormConfigPermission.DOMAIN.toLowerCase(), FormConfigId.Parser.INSTANCE, storage::getFormConfig) + log.info( + "Owned FormConfig permissions deleted: {}", + deletePermissionsOfOwnedInstances( + storage, + FormConfigPermission.DOMAIN.toLowerCase(), + FormConfigId.Parser.INSTANCE, + storage::getFormConfig) ); } @@ -56,10 +72,12 @@ public void execute(Map> parameters, PrintWriter output) th * * @return The number of deleted permissions. */ - public static int deleteQueryPermissionsWithMissingRef(MetaStorage storage, Stream> owners) { + public static int deleteQueryPermissionsWithMissingRef( + MetaStorage storage, + Stream> owners) { int countDeleted = 0; // Do the loop-di-loop - for (Iterator> it = owners.iterator(); it.hasNext(); ) { + for (Iterator> it = owners.iterator(); it.hasNext();) { PermissionOwner owner = it.next(); Set permissions = owner.getPermissions(); for (Permission permission : permissions) { @@ -88,7 +106,8 @@ public static int deleteQueryPermissionsWithMissingRef(MetaStorage storage, Stre } // Create a new Permission that only contains valid references WildcardPermission reducedPermission = new WildcardPermission( - List.of(wpermission.getDomains(), wpermission.getAbilities(), validRef), wpermission.getCreationTime()); + List.of(wpermission.getDomains(), wpermission.getAbilities(), validRef), + wpermission.getCreationTime()); owner.addPermission(reducedPermission); } @@ -105,9 +124,9 @@ public static int deleteQueryPermissionsWithMissingRef(MetaStorage storage, Stre private static WildcardPermission getAsWildcardPermission(Permission permission) { if (!(permission instanceof WildcardPermission)) { log.warn( - "Encountered the permission type {} that is not handled by this routine. Permission was: {}", - permission.getClass(), - permission + "Encountered the permission type {} that is not handled by this routine. Permission was: {}", + permission.getClass(), + permission ); return null; } @@ -120,12 +139,12 @@ private static WildcardPermission getAsWildcardPermission(Permission permission) * @return The number of deleted permissions. */ public static & Owned, ID extends Id> int deletePermissionsOfOwnedInstances( - MetaStorage storage, - String permissionDomain, - IdUtil.Parser idParser, - Function instanceStorageExtractor) { + MetaStorage storage, + String permissionDomain, + IdUtil.Parser idParser, + Function instanceStorageExtractor) { int countDeleted = 0; - for (Iterator it = storage.getAllUsers().iterator(); it.hasNext(); ) { + for (Iterator it = storage.getAllUsers().iterator(); it.hasNext();) { User user = it.next(); Set permissions = user.getPermissions(); for (Permission permission : permissions) { @@ -145,24 +164,26 @@ private static WildcardPermission getAsWildcardPermission(Permission permission) ID executionId; try { executionId = idParser.parse(wpermission.getInstances().iterator().next()); - } - catch (Exception e) { + } catch (Exception e) { log.warn("Unable to parse an id from permission instance. Permission was: {}", wpermission); continue; } E execution = instanceStorageExtractor.apply(executionId); if (execution == null) { - log.trace("The execution referenced in permission {} does not exist. Skipping permission", wpermission); + log.trace( + "The execution referenced in permission {} does not exist. Skipping permission", + wpermission); continue; } if (!user.isOwner(execution)) { - log.trace("The user is not owner of the instance. Keeping the permission. User: {}, Owner: {}, Instance: {}, Permission: {}", - user.getId(), - execution.getOwner(), - execution.getId(), - wpermission + log.trace( + "The user is not owner of the instance. Keeping the permission. User: {}, Owner: {}, Instance: {}, Permission: {}", + user.getId(), + execution.getOwner(), + execution.getId(), + wpermission ); continue; } diff --git a/backend/src/main/java/com/bakdata/conquery/tasks/QueryCleanupTask.java b/backend/src/main/java/com/bakdata/conquery/tasks/QueryCleanupTask.java index 9d25713a049..6c40184b005 100644 --- a/backend/src/main/java/com/bakdata/conquery/tasks/QueryCleanupTask.java +++ b/backend/src/main/java/com/bakdata/conquery/tasks/QueryCleanupTask.java @@ -1,138 +1,146 @@ -package com.bakdata.conquery.tasks; - -import java.io.PrintWriter; -import java.time.Duration; -import java.time.LocalDateTime; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.models.execution.ManagedExecution; -import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; -import io.dropwizard.servlets.tasks.Task; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.ArrayUtils; - -/** - * Dropwizard Task deleting queries that are not used anymore. Defined as: - * - not named - * - older than 30 days - * - is not shared - * - no tags - * - not referenced by other queries - */ -@Slf4j -public class QueryCleanupTask extends Task { - - public static final String EXPIRATION_PARAM = "expiration"; - public static final String DRY_RUN_PARAM = "dry-run"; - - private final MetaStorage storage; - private final Duration queryExpiration; - - public QueryCleanupTask(MetaStorage storage, Duration queryExpiration) { - super("query-cleanup"); - this.storage = storage; - this.queryExpiration = queryExpiration; - } - - @Override - public void execute(Map> parameters, PrintWriter output) throws Exception { - - boolean dryRun = Optional.ofNullable(parameters.get(DRY_RUN_PARAM)).filter(Predicate.not(List::isEmpty)).map(List::getFirst).map(Boolean::parseBoolean).orElse(false); - - Duration queryExpiration = this.queryExpiration; - - if (parameters.containsKey(EXPIRATION_PARAM)) { - if (parameters.get(EXPIRATION_PARAM).size() > 1) { - log.warn("Will not respect more than one expiration time. Have `{}`", parameters.get(EXPIRATION_PARAM)); - } - - queryExpiration = Duration.parse(parameters.get(EXPIRATION_PARAM).getFirst()); - } - - if (queryExpiration == null) { - throw new IllegalArgumentException("Query Expiration may not be null"); - } - - log.info("Starting deletion of queries older than {} of {}", queryExpiration, storage.getAllExecutions().count()); - - // Iterate for as long as no changes are needed (this is because queries can be referenced by other queries) - while (true) { - final Set requiredQueries = new HashSet<>(); - - final Set toDelete = new HashSet<>(); - - for (ManagedExecution execution : storage.getAllExecutions().toList()) { - - // Gather all referenced queries via reused checker. - requiredQueries.addAll(execution.getSubmitted().collectRequiredQueries()); - - if (execution.isSystem()) { - // System Queries will always be deleted. - toDelete.add(execution); - continue; - } - - log.trace("{} is not system", execution.getId()); - - if (execution.isShared()) { - continue; - } - log.trace("{} is not shared", execution.getId()); - - - if (ArrayUtils.isNotEmpty(execution.getTags())) { - continue; - } - log.trace("{} has no tags", execution.getId()); - - if (execution.getLabel() != null && !execution.isAutoLabeled()) { - continue; - } - log.trace("{} has no custom label", execution.getId()); - - - if (LocalDateTime.now().minus(queryExpiration).isBefore(execution.getCreationTime())) { - continue; - } - log.trace("{} is not older than {}.", execution.getId(), queryExpiration); - - toDelete.add(execution); - } - - // remove all queries referenced in reused queries. - final Collection referenced = - requiredQueries.stream() - .map(storage::getExecution) - .collect(Collectors.toSet()); - - toDelete.removeAll(referenced); - - - if (toDelete.isEmpty()) { - log.info("No queries to delete"); - break; - } - - if (dryRun) { - log.info("Dry Run: Cleanup would delete {} Executions", toDelete.size()); - return; - } - - log.info("Deleting {} Executions", toDelete.size()); - - for (ManagedExecution execution : toDelete) { - log.trace("Deleting Execution[{}]", execution.getId()); - storage.removeExecution(execution.getId()); - } - - } - } -} +package com.bakdata.conquery.tasks; + +import java.io.PrintWriter; +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.models.execution.ManagedExecution; +import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; +import io.dropwizard.servlets.tasks.Task; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.ArrayUtils; + +/** + * Dropwizard Task deleting queries that are not used anymore. Defined as: + * - not named + * - older than 30 days + * - is not shared + * - no tags + * - not referenced by other queries + */ +@Slf4j +public class QueryCleanupTask extends Task { + + public static final String EXPIRATION_PARAM = "expiration"; + public static final String DRY_RUN_PARAM = "dry-run"; + + private final MetaStorage storage; + private final Duration queryExpiration; + + public QueryCleanupTask(MetaStorage storage, Duration queryExpiration) { + super("query-cleanup"); + this.storage = storage; + this.queryExpiration = queryExpiration; + } + + @Override + public void execute(Map> parameters, PrintWriter output) throws Exception { + + boolean dryRun = Optional.ofNullable(parameters.get(DRY_RUN_PARAM)) + .filter(Predicate.not(List::isEmpty)) + .map( + List::getFirst) + .map(Boolean::parseBoolean) + .orElse(false); + + Duration queryExpiration = this.queryExpiration; + + if (parameters.containsKey(EXPIRATION_PARAM)) { + if (parameters.get(EXPIRATION_PARAM).size() > 1) { + log.warn("Will not respect more than one expiration time. Have `{}`", parameters.get(EXPIRATION_PARAM)); + } + + queryExpiration = Duration.parse(parameters.get(EXPIRATION_PARAM).getFirst()); + } + + if (queryExpiration == null) { + throw new IllegalArgumentException("Query Expiration may not be null"); + } + + log.info( + "Starting deletion of queries older than {} of {}", + queryExpiration, + storage.getAllExecutions().count()); + + // Iterate for as long as no changes are needed (this is because queries can be referenced by other queries) + while (true) { + final Set requiredQueries = new HashSet<>(); + + final Set toDelete = new HashSet<>(); + + for (ManagedExecution execution : storage.getAllExecutions().toList()) { + + // Gather all referenced queries via reused checker. + requiredQueries.addAll(execution.getSubmitted().collectRequiredQueries()); + + if (execution.isSystem()) { + // System Queries will always be deleted. + toDelete.add(execution); + continue; + } + + log.trace("{} is not system", execution.getId()); + + if (execution.isShared()) { + continue; + } + log.trace("{} is not shared", execution.getId()); + + + if (ArrayUtils.isNotEmpty(execution.getTags())) { + continue; + } + log.trace("{} has no tags", execution.getId()); + + if (execution.getLabel() != null && !execution.isAutoLabeled()) { + continue; + } + log.trace("{} has no custom label", execution.getId()); + + + if (LocalDateTime.now().minus(queryExpiration).isBefore(execution.getCreationTime())) { + continue; + } + log.trace("{} is not older than {}.", execution.getId(), queryExpiration); + + toDelete.add(execution); + } + + // remove all queries referenced in reused queries. + final Collection referenced = requiredQueries.stream() + .map(storage::getExecution) + .collect( + Collectors.toSet()); + + toDelete.removeAll(referenced); + + + if (toDelete.isEmpty()) { + log.info("No queries to delete"); + break; + } + + if (dryRun) { + log.info("Dry Run: Cleanup would delete {} Executions", toDelete.size()); + return; + } + + log.info("Deleting {} Executions", toDelete.size()); + + for (ManagedExecution execution : toDelete) { + log.trace("Deleting Execution[{}]", execution.getId()); + storage.removeExecution(execution.getId()); + } + + } + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/tasks/ReportConsistencyTask.java b/backend/src/main/java/com/bakdata/conquery/tasks/ReportConsistencyTask.java index 533c9fdc8cd..9c652470909 100644 --- a/backend/src/main/java/com/bakdata/conquery/tasks/ReportConsistencyTask.java +++ b/backend/src/main/java/com/bakdata/conquery/tasks/ReportConsistencyTask.java @@ -10,17 +10,20 @@ public class ReportConsistencyTask extends Task { - private final ClusterState clusterState; + private final ClusterState clusterState; - public ReportConsistencyTask(ClusterState clusterState) { - super("report-consistency"); - this.clusterState = clusterState; - } + public ReportConsistencyTask(ClusterState clusterState) { + super("report-consistency"); + this.clusterState = clusterState; + } - @Override - public void execute(Map> parameters, PrintWriter output) throws Exception { - clusterState.getWorkerHandlers().values().stream() - .flatMap(ns -> ns.getWorkers().stream()) - .forEach(worker -> worker.send(new RequestConsistency())); - } + @Override + public void execute(Map> parameters, PrintWriter output) throws Exception { + clusterState.getWorkerHandlers() + .values() + .stream() + .flatMap(ns -> ns.getWorkers().stream()) + .forEach( + worker -> worker.send(new RequestConsistency())); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/util/AuthUtil.java b/backend/src/main/java/com/bakdata/conquery/util/AuthUtil.java index 7dc039b57e7..9ca1e6ab2b3 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/AuthUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/util/AuthUtil.java @@ -16,7 +16,10 @@ public class AuthUtil { /** * When the execution finishes, the service user and all its executions are cleaned up. */ - public synchronized void cleanUpUserAndBelongings(User user, MetaStorage storage, DatasetRegistry datasetRegistry) { + public synchronized void cleanUpUserAndBelongings( + User user, + MetaStorage storage, + DatasetRegistry datasetRegistry) { log.debug("BEGIN Removing user '{}' and it's assets", user); @@ -38,7 +41,7 @@ public synchronized void cleanUpUserAndBelongings(User user, MetaStorage storage continue; } log.trace("Cleaning execution '{}' for user '{}'", exec, user.getId()); - try{ + try { ExecutionManager executionManager = datasetRegistry.get(exec.getDataset()).getExecutionManager(); executionManager.clearQueryResults(exec); @@ -51,15 +54,12 @@ public synchronized void cleanUpUserAndBelongings(User user, MetaStorage storage log.debug("Removed {} form configs and {} executions for user '{}'", countForms, countExecs, user); - storage.getAllGroups() - .filter(group -> group.containsUser(user.getId())) - .forEach(group -> - { - group.removeMember(user.getId()); - group.updateStorage(); - log.debug("Removed user '{}' from group '{}'", user.getId(), group.getId()); - } - ); + storage.getAllGroups().filter(group -> group.containsUser(user.getId())).forEach(group -> { + group.removeMember(user.getId()); + group.updateStorage(); + log.debug("Removed user '{}' from group '{}'", user.getId(), group.getId()); + } + ); storage.removeUser(user.getId()); diff --git a/backend/src/main/java/com/bakdata/conquery/util/CollectionsUtil.java b/backend/src/main/java/com/bakdata/conquery/util/CollectionsUtil.java index 121d9970834..4eb7c201c11 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/CollectionsUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/util/CollectionsUtil.java @@ -1,10 +1,8 @@ package com.bakdata.conquery.util; import java.util.Collections; -import java.util.Map; import java.util.Set; -import it.unimi.dsi.fastutil.objects.Object2ObjectArrayMap; import it.unimi.dsi.fastutil.objects.ObjectArraySet; import lombok.experimental.UtilityClass; diff --git a/backend/src/main/java/com/bakdata/conquery/util/ConqueryEscape.java b/backend/src/main/java/com/bakdata/conquery/util/ConqueryEscape.java index 69390aa5088..8caf4991627 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/ConqueryEscape.java +++ b/backend/src/main/java/com/bakdata/conquery/util/ConqueryEscape.java @@ -80,24 +80,23 @@ private static String escapeRequired(byte[] bytes, int index, ByteArrayOutputStr for (int i = index; i < bytes.length; i++) { if (dontNeedEncoding(bytes[i])) { baos.write(bytes[i]); - } - else { + } else { encode(bytes[i], baos); } } return baos.toString(StandardCharsets.US_ASCII); } - + public static String unescape(@NonNull String word) { - if(word.isEmpty()) { + if (word.isEmpty()) { return word; } - + byte[] bytes = word.getBytes(StandardCharsets.US_ASCII); for (int i = 0; i < bytes.length; i++) { byte aByte = bytes[i]; - if (aByte == ESCAPER){ + if (aByte == ESCAPER) { continue; } if (dontNeedEncoding(aByte)) { @@ -106,17 +105,16 @@ public static String unescape(@NonNull String word) { log.warn("Unescaped character '{}' at {} in '{}'", aByte, i, word); } - if(!ArrayUtils.contains(bytes, ESCAPER)) { + if (!ArrayUtils.contains(bytes, ESCAPER)) { return word; } - + ByteArrayOutputStream out = new ByteArrayOutputStream(bytes.length); - - for(int i=0;i DATE_CACHE = CacheBuilder.newBuilder() - .weakValues() - .concurrencyLevel(10) - .build(CacheLoader.from(this::tryParseDate)); + .weakValues() + .concurrencyLevel( + 10) + .build(CacheLoader.from(this::tryParseDate)); @JsonIgnore private final List rangeStartEndSeperators; @JsonIgnore private final List dateSetLayouts; @JsonCreator - public DateReader(Set dateParsingFormats, List rangeStartEndSeperators, List dateSetLayouts) { + public DateReader( + Set dateParsingFormats, + List rangeStartEndSeperators, + List dateSetLayouts) { this.dateFormats = dateParsingFormats.stream().map(DateTimeFormatter::ofPattern).collect(Collectors.toList()); this.rangeStartEndSeperators = rangeStartEndSeperators; this.dateSetLayouts = dateSetLayouts; @@ -93,8 +97,7 @@ public CDateRange parseToCDateRange(String value) { CDateRange result = parseToCDateRange(value, sep); lastRangeFormatIndex.set(index); return result; - } - catch (ParsingException e) { + } catch (ParsingException e) { log.trace("Parsing failed for date range `{}` using `{}`", value, sep, e); } } @@ -131,12 +134,18 @@ private CDateRange parseToCDateRange(String value, String sep) { if (parts.length == 2) { return CDateRange.of( - parseToLocalDate(parts[0]), - parseToLocalDate(parts[1]) + parseToLocalDate(parts[0]), + parseToLocalDate(parts[1]) ); } - throw ParsingException.of(value, String.format("DateRange: Unexpected length of Parts (%d) for `%s` using sep=`%s`", parts.length, value, sep)); + throw ParsingException.of( + value, + String.format( + "DateRange: Unexpected length of Parts (%d) for `%s` using sep=`%s`", + parts.length, + value, + sep)); } /** @@ -177,8 +186,7 @@ public CDateSet parseToCDateSet(String value) { final CDateSet result = sep.parse(value, this); lastDateSetLayoutIndex.set(index); return result; - } - catch (ParsingException e) { + } catch (ParsingException e) { log.trace("Parsing failed for date set '{}' with pattern '{}'", value, sep, e); } } @@ -205,8 +213,7 @@ private LocalDate tryParseDate(String value) { lastDateFormatIndex.set(index); return res; - } - catch (DateTimeParseException e) { + } catch (DateTimeParseException e) { log.trace("Failed to parse date `{}` using `{}`", value, format, e); } } diff --git a/backend/src/main/java/com/bakdata/conquery/util/NumberParsing.java b/backend/src/main/java/com/bakdata/conquery/util/NumberParsing.java index 0dda77cedab..23ddc931430 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/NumberParsing.java +++ b/backend/src/main/java/com/bakdata/conquery/util/NumberParsing.java @@ -5,13 +5,13 @@ import com.bakdata.conquery.models.exceptions.ParsingException; public class NumberParsing { - + //see #150 parse with ConqueryConfig.getInstance().getLocale() - + public static long parseLong(String value) throws ParsingException { try { - return Long.valueOf(value); - } catch(Exception e) { + return Long.parseLong(value); + } catch (Exception e) { throw ParsingException.of(value, "long", e); } } @@ -19,7 +19,7 @@ public static long parseLong(String value) throws ParsingException { public static double parseDouble(String value) throws ParsingException { try { return Double.parseDouble(value); - } catch(Exception e) { + } catch (Exception e) { throw ParsingException.of(value, "double", e); } } @@ -27,11 +27,11 @@ public static double parseDouble(String value) throws ParsingException { public static BigDecimal parseBig(String value) throws ParsingException { try { return new BigDecimal(value); - } catch(Exception e) { + } catch (Exception e) { throw ParsingException.of(value, "BigDecimal", e); } } - + public static BigDecimal parseMoney(String value) throws ParsingException { try { return parseBig(value).stripTrailingZeros(); @@ -39,4 +39,4 @@ public static BigDecimal parseMoney(String value) throws ParsingException { throw ParsingException.of(value, "money", e); } } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/util/PrettyPrinter.java b/backend/src/main/java/com/bakdata/conquery/util/PrettyPrinter.java index 69512dfe085..184d497bc32 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/PrettyPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/util/PrettyPrinter.java @@ -16,25 +16,24 @@ public class PrettyPrinter { private static final Indenter INDENTER = new DefaultIndenter("\t", "\n"); private static final DefaultPrettyPrinter PRETTY_PRINTER = new DefaultPrettyPrinter() { private static final long serialVersionUID = 1L; + @Override public DefaultPrettyPrinter withSeparators(Separators separators) { - _separators = separators; - _objectFieldValueSeparatorWithSpaces = separators.getObjectFieldValueSeparator() + " "; - return this; - } - } - .withSeparators(new Separators()) - .withArrayIndenter(INDENTER) - .withObjectIndenter(INDENTER) - .withSpacesInObjectEntries(); + _separators = separators; + _objectFieldValueSeparatorWithSpaces = separators.getObjectFieldValueSeparator() + " "; + return this; + } + }.withSeparators(new Separators()) + .withArrayIndenter(INDENTER) + .withObjectIndenter( + INDENTER) + .withSpacesInObjectEntries(); public static String print(String json) throws JsonProcessingException, IOException { try { - return Jackson.MAPPER - .setDefaultPrettyPrinter(PRETTY_PRINTER) - .writerWithDefaultPrettyPrinter() - .writeValueAsString(Jackson.MAPPER.readTree(json)); - } catch(Exception e) { + return Jackson.MAPPER.setDefaultPrettyPrinter( + PRETTY_PRINTER).writerWithDefaultPrettyPrinter().writeValueAsString(Jackson.MAPPER.readTree(json)); + } catch (Exception e) { log.warn("Could not pretty print:\n{}", json); return json; } diff --git a/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java b/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java index 7198fcc577e..f62c65047a1 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java +++ b/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java @@ -48,11 +48,13 @@ public class QueryUtils { private static final int MAX_CONCEPT_LABEL_CONCAT_LENGTH = 70; + /** * Provides a starting operator for consumer chains, that does nothing. */ public static Consumer getNoOpEntryPoint() { - return (whatever) -> {}; + return (whatever) -> { + }; } /** @@ -65,17 +67,31 @@ public static T getVisitor(List visitors, /** * Create label for children preferring the children's user label if provided. */ - public static String createUserMultiLabel(List elements, String delimiter, String postfix, Locale locale) { - return elements.stream().map(elt -> elt.userLabel(locale)).collect(Collectors.joining(delimiter,"", postfix)); + public static String createUserMultiLabel( + List elements, + String delimiter, + String postfix, + Locale locale) { + return elements.stream().map(elt -> elt.userLabel(locale)).collect(Collectors.joining(delimiter, "", postfix)); } + /** * Create label for children the children's default label ignoring user provided label. */ - public static String createDefaultMultiLabel(List elements, String delimiter, String postfix, Locale locale) { - return elements.stream().map(elt -> elt.defaultLabel(locale)).collect(Collectors.joining(delimiter, "", postfix)); + public static String createDefaultMultiLabel( + List elements, + String delimiter, + String postfix, + Locale locale) { + return elements.stream() + .map(elt -> elt.defaultLabel(locale)) + .collect( + Collectors.joining(delimiter, "", postfix)); } - public static QueryExecutionContext determineDateAggregatorForContext(QueryExecutionContext ctx, Supplier>> altValidityDateAggregator) { + public static QueryExecutionContext determineDateAggregatorForContext( + QueryExecutionContext ctx, + Supplier>> altValidityDateAggregator) { if (ctx.getQueryDateAggregator().isPresent()) { return ctx; } @@ -85,9 +101,9 @@ public static QueryExecutionContext determineDateAggregatorForContext(QueryExecu public static String makeQueryLabel(final Visitable query, PrintSettings cfg, ManagedExecutionId id) { final StringBuilder sb = new StringBuilder(); - final Map, List> sortedContents = - Visitable.stream(query) - .collect(Collectors.groupingBy(Visitable::getClass)); + final Map, List> sortedContents = Visitable.stream(query) + .collect( + Collectors.groupingBy(Visitable::getClass)); int sbStartSize = sb.length(); @@ -118,15 +134,17 @@ public static String makeQueryLabel(final Visitable query, PrintSettings cfg, Ma final AtomicInteger length = new AtomicInteger(); sortedContents.get(CQConcept.class) - .stream() - .map(CQConcept.class::cast) + .stream() + .map(CQConcept.class::cast) - .map(c -> makeLabelWithRootAndChild(c, cfg)) - .filter(Predicate.not(Strings::isNullOrEmpty)) - .distinct() + .map(c -> makeLabelWithRootAndChild(c, cfg)) + .filter( + Predicate.not(Strings::isNullOrEmpty)) + .distinct() - .takeWhile(elem -> length.addAndGet(elem.length()) < MAX_CONCEPT_LABEL_CONCAT_LENGTH) - .forEach(label -> sb.append(label).append(" ")); + .takeWhile(elem -> length.addAndGet(elem.length()) < MAX_CONCEPT_LABEL_CONCAT_LENGTH) + .forEach( + label -> sb.append(label).append(" ")); // Last entry will output one Space that we don't want if (!sb.isEmpty()) { @@ -158,7 +176,7 @@ private static String makeLabelWithRootAndChild(CQConcept cqConcept, PrintSettin // Concat everything with dashes return label.replace(" ", "-"); } - + /** * Checks if the query requires to resolve external ids. */ @@ -191,7 +209,7 @@ public static class OnlyReusingChecker implements QueryVisitor { @Override public void accept(Visitable element) { - if(containsOthersElements){ + if (containsOthersElements) { return; } @@ -201,8 +219,7 @@ public void accept(Visitable element) { if (reusedQuery == null) { reusedQuery = (CQReusedQuery) element; - } - else { + } else { containsOthersElements = true; } return; @@ -250,16 +267,16 @@ public static class AvailableSecondaryIdCollector implements QueryVisitor { @Override public void accept(Visitable element) { - if(element instanceof final CQConcept cqConcept){ + if (element instanceof final CQConcept cqConcept) { // Excluded Concepts are not available - if(cqConcept.isExcludeFromSecondaryId()){ + if (cqConcept.isExcludeFromSecondaryId()) { return; } for (Connector connector : cqConcept.getConcept().getConnectors()) { for (Column column : connector.getResolvedTable().getColumns()) { - if(column.getSecondaryId() == null){ + if (column.getSecondaryId() == null) { continue; } diff --git a/backend/src/main/java/com/bakdata/conquery/util/ResourceUtil.java b/backend/src/main/java/com/bakdata/conquery/util/ResourceUtil.java index a95647dd156..be7a42ccb7f 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/ResourceUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/util/ResourceUtil.java @@ -1,14 +1,17 @@ -package com.bakdata.conquery.util; - -import com.google.common.base.Preconditions; -import lombok.experimental.UtilityClass; - -@UtilityClass -public class ResourceUtil { - - public static String wrapAsUriTemplate(String templateString) { - Preconditions.checkArgument(templateString.matches("^[a-z-_A-Z]+(: .+)?"), "Provided template string does not match allowed format (see https://docs.oracle.com/cd/E19798-01/821-1841/6nmq2cp26/index.html). Was: %s", templateString); - return "{" + templateString + "}"; - } - -} +package com.bakdata.conquery.util; + +import com.google.common.base.Preconditions; +import lombok.experimental.UtilityClass; + +@UtilityClass +public class ResourceUtil { + + public static String wrapAsUriTemplate(String templateString) { + Preconditions.checkArgument( + templateString.matches("^[a-z-_A-Z]+(: .+)?"), + "Provided template string does not match allowed format (see https://docs.oracle.com/cd/E19798-01/821-1841/6nmq2cp26/index.html). Was: %s", + templateString); + return "{" + templateString + "}"; + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/util/SMTPAppenderFactory.java b/backend/src/main/java/com/bakdata/conquery/util/SMTPAppenderFactory.java index e912db7fbfa..6e068a1878a 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/SMTPAppenderFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/util/SMTPAppenderFactory.java @@ -27,7 +27,12 @@ public class SMTPAppenderFactory extends AbstractAppenderFactory { @Override - public Appender build(LoggerContext context, String applicationName, LayoutFactory layoutFactory, LevelFilterFactory levelFilterFactory, AsyncAppenderFactory asyncAppenderFactory) { + public Appender build( + LoggerContext context, + String applicationName, + LayoutFactory layoutFactory, + LevelFilterFactory levelFilterFactory, + AsyncAppenderFactory asyncAppenderFactory) { final SMTPAppender appender = new SMTPAppender(); appender.setName("smtp-appender"); diff --git a/backend/src/main/java/com/bakdata/conquery/util/TablePrimaryColumnUtil.java b/backend/src/main/java/com/bakdata/conquery/util/TablePrimaryColumnUtil.java index 8abfe717061..e01cd132d74 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/TablePrimaryColumnUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/util/TablePrimaryColumnUtil.java @@ -6,8 +6,6 @@ import com.bakdata.conquery.models.datasets.Table; import org.jooq.Field; -import java.util.Objects; - public class TablePrimaryColumnUtil { public static Field findPrimaryColumn(Table table, String defaultPrimaryColumn) { diff --git a/backend/src/main/java/com/bakdata/conquery/util/VariableDefaultValue.java b/backend/src/main/java/com/bakdata/conquery/util/VariableDefaultValue.java index 89a2b360f29..0608707b2bb 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/VariableDefaultValue.java +++ b/backend/src/main/java/com/bakdata/conquery/util/VariableDefaultValue.java @@ -14,4 +14,4 @@ @Target(FIELD) public @interface VariableDefaultValue { -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/util/VersionInfo.java b/backend/src/main/java/com/bakdata/conquery/util/VersionInfo.java index e1d908a7acb..f6b91bccc22 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/VersionInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/util/VersionInfo.java @@ -42,14 +42,12 @@ private VersionInfo() { ZonedDateTime parsedBuildTime = null; try { parsedBuildTime = ZonedDateTime.parse(timeProp); - } - catch (Exception e) { + } catch (Exception e) { log.error("Could not parse date time from git.properties", e); } buildTime = parsedBuildTime; projectVersion = properties.getProperty("project.version"); - } - catch (Exception e) { + } catch (Exception e) { throw new IllegalStateException("Could not read git properties information", e); } } @@ -66,4 +64,4 @@ public List getVersions() { public VersionContainer setFormBackendVersion(VersionContainer version) { return formBackendVersions.put(version.name(), version); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/util/functions/ChainableUnaryOperator.java b/backend/src/main/java/com/bakdata/conquery/util/functions/ChainableUnaryOperator.java index 8ef69962a22..e41190ec135 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/functions/ChainableUnaryOperator.java +++ b/backend/src/main/java/com/bakdata/conquery/util/functions/ChainableUnaryOperator.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.util.functions; +import java.util.List; import java.util.Objects; import java.util.function.Function; import java.util.function.UnaryOperator; @@ -7,47 +8,47 @@ /** * Adds convenience methods to chain {@link UnaryOperator}s and still have them typed as * an {@link UnaryOperator} instead of an {@link Function}. - * This way the chain can be used with {@link List::replaceAll} for efficient replacement. - * + * This way the chain can be used with {@link List#replaceAll(UnaryOperator)} ceAll} for efficient replacement. + * * @param The input and output type of the operator */ @FunctionalInterface -public interface ChainableUnaryOperator extends UnaryOperator{ - - /** - * Returns a composed function that first applies the {@code before} - * function to its input, and then applies this function to the result. - * If evaluation of either function throws an exception, it is relayed to - * the caller of the composed function. - * - * @param before the function to apply before this function is applied - * @return a composed function that first applies the {@code before} - * function and then applies this function - * @throws NullPointerException if before is null - * - * @see #unaryAndThen(Function) - */ - default ChainableUnaryOperator unaryCompose(Function before) { - Objects.requireNonNull(before); - return (T t) -> apply(before.apply(t)); - } +public interface ChainableUnaryOperator extends UnaryOperator { - /** - * Returns a composed function that first applies this function to - * its input, and then applies the {@code after} function to the result. - * If evaluation of either function throws an exception, it is relayed to - * the caller of the composed function. - * - * @param after the function to apply after this function is applied - * @return a composed function that first applies this function and then - * applies the {@code after} function - * @throws NullPointerException if after is null - * - * @see #unaryCompose(Function) - */ - default ChainableUnaryOperator unaryAndThen(Function after) { - Objects.requireNonNull(after); - return (T t) -> after.apply(apply(t)); - } + /** + * Returns a composed function that first applies the {@code before} + * function to its input, and then applies this function to the result. + * If evaluation of either function throws an exception, it is relayed to + * the caller of the composed function. + * + * @param before the function to apply before this function is applied + * @return a composed function that first applies the {@code before} + * function and then applies this function + * @throws NullPointerException if before is null + * + * @see #unaryAndThen(Function) + */ + default ChainableUnaryOperator unaryCompose(Function before) { + Objects.requireNonNull(before); + return (T t) -> apply(before.apply(t)); + } + + /** + * Returns a composed function that first applies this function to + * its input, and then applies the {@code after} function to the result. + * If evaluation of either function throws an exception, it is relayed to + * the caller of the composed function. + * + * @param after the function to apply after this function is applied + * @return a composed function that first applies this function and then + * applies the {@code after} function + * @throws NullPointerException if after is null + * + * @see #unaryCompose(Function) + */ + default ChainableUnaryOperator unaryAndThen(Function after) { + Objects.requireNonNull(after); + return (T t) -> after.apply(apply(t)); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/util/io/ChunkingOutputStream.java b/backend/src/main/java/com/bakdata/conquery/util/io/ChunkingOutputStream.java index fa4f0598934..6a12cb4228e 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/io/ChunkingOutputStream.java +++ b/backend/src/main/java/com/bakdata/conquery/util/io/ChunkingOutputStream.java @@ -56,18 +56,17 @@ public void flush() throws IOException { public void close() throws IOException { write(); } - + private void write() { if (off > 0) { - if(off == size) { + if (off == size) { consumer.accept(buf); buf = null; - } - else { + } else { consumer.accept(Arrays.copyOf(buf, off)); } off = 0; - + } } } diff --git a/backend/src/main/java/com/bakdata/conquery/util/io/Cloner.java b/backend/src/main/java/com/bakdata/conquery/util/io/Cloner.java index 84cf314b164..2db7ea816a9 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/io/Cloner.java +++ b/backend/src/main/java/com/bakdata/conquery/util/io/Cloner.java @@ -21,27 +21,24 @@ public static ConqueryConfig clone(ConqueryConfig config, Map, T> i injectableHolder.add(injectable.getKey(), injectable.getValue()); } ConqueryConfig clone = mapper.readValue( - mapper.writeValueAsBytes(config), - ConqueryConfig.class + mapper.writeValueAsBytes(config), + ConqueryConfig.class ); clone.setLoggingFactory(config.getLoggingFactory()); return clone; - } - catch (IOException e) { + } catch (IOException e) { throw new IllegalStateException("Failed to clone a conquery config " + config, e); } } public static T clone(T element, Injectable injectable, Class valueType) { try { - return injectable - .injectIntoNew(Jackson.BINARY_MAPPER) - .readValue( - Jackson.BINARY_MAPPER.writeValueAsBytes(element), - valueType - ); - } - catch (IOException e) { + return injectable.injectIntoNew(Jackson.BINARY_MAPPER) + .readValue( + Jackson.BINARY_MAPPER.writeValueAsBytes(element), + valueType + ); + } catch (IOException e) { throw new IllegalStateException("Failed to clone the CQElement " + element, e); } } diff --git a/backend/src/main/java/com/bakdata/conquery/util/io/ConqueryFileUtil.java b/backend/src/main/java/com/bakdata/conquery/util/io/ConqueryFileUtil.java index 910ce073f62..03ef8cc7a28 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/io/ConqueryFileUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/util/io/ConqueryFileUtil.java @@ -6,9 +6,9 @@ import java.nio.file.Path; public class ConqueryFileUtil { - + public static final Path TEMPORARY_DIRECTORY; - + static { try { TEMPORARY_DIRECTORY = Files.createTempDirectory("conquery_"); diff --git a/backend/src/main/java/com/bakdata/conquery/util/io/EndCheckableInputStream.java b/backend/src/main/java/com/bakdata/conquery/util/io/EndCheckableInputStream.java index 00a6252b030..b0a79807079 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/io/EndCheckableInputStream.java +++ b/backend/src/main/java/com/bakdata/conquery/util/io/EndCheckableInputStream.java @@ -12,7 +12,7 @@ public EndCheckableInputStream(InputStream in) { public boolean isAtEnd() throws IOException { int b = this.read(); - if(b == -1) { + if (b == -1) { return true; } this.unread(b); diff --git a/backend/src/main/java/com/bakdata/conquery/util/io/FileUtil.java b/backend/src/main/java/com/bakdata/conquery/util/io/FileUtil.java index 4897a0c7155..44cfc358997 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/io/FileUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/util/io/FileUtil.java @@ -21,26 +21,25 @@ public class FileUtil { public static final Pattern SAVE_FILENAME_REPLACEMENT_MATCHER = Pattern.compile("[^a-zA-Z0-9äÄöÖüÜß .\\-]"); - public static String makeSafeFileName(String label) { - return SAVE_FILENAME_REPLACEMENT_MATCHER.matcher(label).replaceAll("_"); - } + public static String makeSafeFileName(String label) { + return SAVE_FILENAME_REPLACEMENT_MATCHER.matcher(label).replaceAll("_"); + } - public void deleteRecursive(Path path) throws IOException { - Files.walkFileTree((path), - new SimpleFileVisitor<>() { - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { - Files.delete(dir); - return FileVisitResult.CONTINUE; - } + public void deleteRecursive(Path path) throws IOException { + Files.walkFileTree((path), new SimpleFileVisitor<>() { + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - Files.delete(file); - return FileVisitResult.CONTINUE; - } - }); - } + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + }); + } /** @@ -67,8 +66,7 @@ public static URI getResolvedUri(@Nullable URI baseUri, URI specificUri) { try { // Check if resolved URI is still a valid URL resolvedURI.toURL(); - } - catch (MalformedURLException e) { + } catch (MalformedURLException e) { throw new IllegalArgumentException("Resolved url is not valid", e); } return resolvedURI; diff --git a/backend/src/main/java/com/bakdata/conquery/util/io/IdColumnUtil.java b/backend/src/main/java/com/bakdata/conquery/util/io/IdColumnUtil.java index 3f1716a3f66..dab2cf81825 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/io/IdColumnUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/util/io/IdColumnUtil.java @@ -23,7 +23,9 @@ public class IdColumnUtil { /** * If a column contains an ID, create a reader for that ID. */ - public static List> getIdReaders(List format, Map idMappers) { + public static List> getIdReaders( + List format, + Map idMappers) { List> out = new ArrayList<>(format.size()); for (int index = 0; index < format.size(); index++) { @@ -44,13 +46,18 @@ public static List> getIdReaders(List for /** * Try to create a {@link FullIdPrinter} for user if they are allowed. If not allowed to read ids, they will receive a pseudomized result instead. */ - public static IdPrinter getIdPrinter(Subject owner, ManagedExecution execution, Namespace namespace, List ids) { + public static IdPrinter getIdPrinter( + Subject owner, + ManagedExecution execution, + Namespace namespace, + List ids) { final int size = (int) ids.stream().filter(ColumnConfig::isPrint).count(); final int pos = IntStream.range(0, ids.size()) - .filter(idx -> ids.get(idx).isPrimaryId()) - .findFirst() - .orElseThrow(); + .filter( + idx -> ids.get(idx).isPrimaryId()) + .findFirst() + .orElseThrow(); if (owner.isPermitted(execution.getDataset(), Ability.PRESERVE_ID)) { // todo(tm): The integration of ids in the sql connector needs to be properly managed diff --git a/backend/src/main/java/com/bakdata/conquery/util/io/LogUtil.java b/backend/src/main/java/com/bakdata/conquery/util/io/LogUtil.java index f1a5c29b6d6..e4e3049e1cd 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/io/LogUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/util/io/LogUtil.java @@ -33,7 +33,7 @@ public static Throwable passExceptionOnTrace(Logger log, Throwable e) { public String printPath(File f) { return printPath(f.toPath()); } - + public String printPath(Path p) { try { return p.toRealPath(LinkOption.NOFOLLOW_LINKS).toString(); diff --git a/backend/src/main/java/com/bakdata/conquery/util/io/ProgressBar.java b/backend/src/main/java/com/bakdata/conquery/util/io/ProgressBar.java index fa6e292dd05..8ed8275d0d0 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/io/ProgressBar.java +++ b/backend/src/main/java/com/bakdata/conquery/util/io/ProgressBar.java @@ -6,18 +6,15 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; -//TODO replace with https://github.com/vdurmont/etaprinter +// TODO replace with https://github.com/vdurmont/etaprinter @Slf4j public class ProgressBar { - + private static final int CHARACTERS = 50; - private static final char[] BAR_CHARACTERS = { - ' ', - '▌', - '█' + private static final char[] BAR_CHARACTERS = {' ', '▌', '█' }; private static final char RIGHT = '▌'; - + private final AtomicLong currentValue = new AtomicLong(0); @Getter private final AtomicLong maxValue; @@ -28,53 +25,54 @@ public ProgressBar(long maxValue) { this.maxValue = new AtomicLong(Math.max(1, maxValue)); this.startTime = System.nanoTime(); } - + public void addCurrentValue(long add) { long current = currentValue.addAndGet(add); - long newPercentage = current*100L/maxValue.get(); + long newPercentage = current * 100L / maxValue.get(); long last = lastPercentage.getAndSet(newPercentage); - if(newPercentage!=last) { + if (newPercentage != last) { print(); } } - + public void addMaxValue(long add) { maxValue.addAndGet(add); } - + public void done() { StringBuilder sb = new StringBuilder(); sb.append(String.valueOf(BAR_CHARACTERS[2]).repeat(CHARACTERS)); sb.append(" 100% DONE"); log.info(sb.toString()); } - + private void print() { StringBuilder sb = new StringBuilder(); long last = lastPercentage.get(); long remaining = last; - - for(int i=0;i max.get()) { - log.warn("Progress({}) + ChildProgressReserve({}) + Steps({}) is bigger than the maximum Progress({}). There might be to many reports in the code.", innerProgress, reservedForChildren, steps, max); + log.warn( + "Progress({}) + ChildProgressReserve({}) + Steps({}) is bigger than the maximum Progress({}). There might be to many reports in the code.", + innerProgress, + reservedForChildren, + steps, + max); return; } @@ -133,14 +143,14 @@ public void done() { } if (getAbsoluteProgress() < getAbsoluteMax()) { - log.trace("Done was called before all steps were been reported. There might be missing reporting steps in the code."); + log.trace( + "Done was called before all steps were been reported. There might be missing reporting steps in the code."); } innerProgress.set(max.get() - reservedForChildren.get()); } - - + @JsonValue public ImmutableProgressReporter toImmutable() { return new ImmutableProgressReporter(this); diff --git a/backend/src/main/java/com/bakdata/conquery/util/progressreporter/ProgressReporterUtil.java b/backend/src/main/java/com/bakdata/conquery/util/progressreporter/ProgressReporterUtil.java index 2a92670fded..c387417091c 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/progressreporter/ProgressReporterUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/util/progressreporter/ProgressReporterUtil.java @@ -15,40 +15,56 @@ @Slf4j public class ProgressReporterUtil { - private final static DateTimeFormatter TIME_FORMATTER = new DateTimeFormatterBuilder() - .appendValue(ChronoField.HOUR_OF_DAY, 2).appendLiteral("h ").appendValue(ChronoField.MINUTE_OF_HOUR, 2) - .appendLiteral("m ").appendValue(ChronoField.SECOND_OF_MINUTE, 2).appendLiteral("s ").toFormatter(); + private final static DateTimeFormatter TIME_FORMATTER = new DateTimeFormatterBuilder().appendValue( + ChronoField.HOUR_OF_DAY, + 2) + .appendLiteral("h ") + .appendValue(ChronoField.MINUTE_OF_HOUR, 2) + .appendLiteral("m ") + .appendValue( + ChronoField.SECOND_OF_MINUTE, + 2) + .appendLiteral("s ") + .toFormatter(); /* package */ final static String UNKNOWN = "unknown "; /* package */ final static String MAX_PROGRESS = "done"; - /* package */ String buildProgressReportString(boolean done, long progress, long maxProgress, long elapsedMillis, long waitedMillis) { + /* package */ String buildProgressReportString( + boolean done, + long progress, + long maxProgress, + long elapsedMillis, + long waitedMillis) { if (done) { return ProgressReporterUtil.MAX_PROGRESS; } - + if (progress == 0 || maxProgress == 0) { // When no estimate can be calculated return String.format( - "waited %s - %3d%% - est. %s", - ProgressReporterUtil.TIME_FORMATTER.format(LocalTime.MIDNIGHT.plus(Duration.ofMillis(waitedMillis))), - 0, - UNKNOWN + "waited %s - %3d%% - est. %s", + ProgressReporterUtil.TIME_FORMATTER.format(LocalTime.MIDNIGHT.plus(Duration.ofMillis(waitedMillis))), + 0, + UNKNOWN ); } - + long remainingProgress = maxProgress - progress; - long estimateMillis = DoubleMath.roundToLong((elapsedMillis / progress) * remainingProgress, RoundingMode.HALF_UP); + long estimateMillis = DoubleMath.roundToLong( + (elapsedMillis / progress) * remainingProgress, + RoundingMode.HALF_UP); Duration estimate = Duration.ofMillis(estimateMillis); - int percent = DoubleMath.roundToInt(progress/maxProgress * 100, RoundingMode.FLOOR); + int percent = DoubleMath.roundToInt(progress / maxProgress * 100, RoundingMode.FLOOR); - return String.format("waited %s - %3d%% - est. %s", - ProgressReporterUtil.TIME_FORMATTER.format(LocalTime.MIDNIGHT.plus(Duration.ofMillis(waitedMillis))), - percent, - ProgressReporterUtil.TIME_FORMATTER.format(LocalTime.MIDNIGHT.plus(estimate))); + return String.format( + "waited %s - %3d%% - est. %s", + ProgressReporterUtil.TIME_FORMATTER.format(LocalTime.MIDNIGHT.plus(Duration.ofMillis(waitedMillis))), + percent, + ProgressReporterUtil.TIME_FORMATTER.format(LocalTime.MIDNIGHT.plus(estimate))); } } diff --git a/backend/src/main/java/com/bakdata/conquery/util/progressreporter/package-info.java b/backend/src/main/java/com/bakdata/conquery/util/progressreporter/package-info.java index 3782a7f9146..124178c33bf 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/progressreporter/package-info.java +++ b/backend/src/main/java/com/bakdata/conquery/util/progressreporter/package-info.java @@ -1,4 +1,4 @@ /** * A generic progress reporter with time estimation. */ -package com.bakdata.conquery.util.progressreporter; \ No newline at end of file +package com.bakdata.conquery.util.progressreporter; diff --git a/backend/src/main/java/com/bakdata/conquery/util/search/SearchProcessor.java b/backend/src/main/java/com/bakdata/conquery/util/search/SearchProcessor.java index d20bbeb0c87..39257091f99 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/search/SearchProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/util/search/SearchProcessor.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.util.search; +import jakarta.validation.constraints.NotNull; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -7,7 +8,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import javax.annotation.CheckForNull; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.apiv1.frontend.FrontendValue; import com.bakdata.conquery.io.storage.NamespaceStorage; @@ -27,12 +27,14 @@ public interface SearchProcessor { * Similar to {@link Managed} but we don't want this to be part of a lifecycle because datasets, etc. might get created after * the lifecycle called {@link Managed#start()} */ - default void start() throws Exception {} + default void start() throws Exception { + } /** * See {@link SearchProcessor#start()} */ - default void stop() throws Exception {} + default void stop() throws Exception { + } /** @@ -74,7 +76,10 @@ static List extractKeywords(FrontendValue value) { /** * Callback for {@link com.bakdata.conquery.models.jobs.UpdateFilterSearchJob} to index values that are present to the manager. */ - void indexManagerResidingSearches(Set managerSearchables, AtomicBoolean cancelledState, ProgressReporter progressReporter) throws InterruptedException; + void indexManagerResidingSearches( + Set managerSearchables, + AtomicBoolean cancelledState, + ProgressReporter progressReporter) throws InterruptedException; /** @@ -95,7 +100,11 @@ static List extractKeywords(FrontendValue value) { * @param pageNumber Pagination: page (zero-based) * @return Found matches */ - ConceptsProcessor.AutoCompleteResult query(SelectFilter filter, @CheckForNull String maybeText, int itemsPerPage, int pageNumber); + ConceptsProcessor.AutoCompleteResult query( + SelectFilter filter, + @CheckForNull String maybeText, + int itemsPerPage, + int pageNumber); /** * Job to run after all ColumnValues are processed by the manager. diff --git a/backend/src/main/java/com/bakdata/conquery/util/search/internal/InternalFilterSearch.java b/backend/src/main/java/com/bakdata/conquery/util/search/internal/InternalFilterSearch.java index 63537fef15f..8611b690a29 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/search/internal/InternalFilterSearch.java +++ b/backend/src/main/java/com/bakdata/conquery/util/search/internal/InternalFilterSearch.java @@ -72,16 +72,21 @@ public class InternalFilterSearch implements SearchProcessor { /** * Cache of all search results on SelectFilters. */ - private final LoadingCache, String>, List> - searchResults = - CacheBuilder.newBuilder().softValues().build(new CacheLoader<>() { + private final LoadingCache, String>, List> searchResults = CacheBuilder + .newBuilder() + .softValues() + .build( + new CacheLoader<>() { @Override public List load(Pair, String> filterAndSearch) { final String searchTerm = filterAndSearch.getValue(); final SelectFilter searchable = filterAndSearch.getKey(); - log.trace("Calculating a new search cache for the term \"{}\" on Searchable[{}]", searchTerm, searchable.getId()); + log.trace( + "Calculating a new search cache for the term \"{}\" on Searchable[{}]", + searchTerm, + searchable.getId()); return topItems(searchable, searchTerm); } @@ -92,14 +97,17 @@ public List load(Pair, String> filterAndSearch) { * Cache of raw listing of values on a filter. * We use Cursor here to reduce strain on memory and increase response time. */ - private final LoadingCache, CursorAndLength> listResults = CacheBuilder.newBuilder().softValues().build(new CacheLoader<>() { - @Override - public CursorAndLength load(SelectFilter searchable) { - log.trace("Creating cursor for `{}`", searchable.getId()); - return new CursorAndLength(listAllValues(searchable), getTotal(searchable)); - } + private final LoadingCache, CursorAndLength> listResults = CacheBuilder.newBuilder() + .softValues() + .build( + new CacheLoader<>() { + @Override + public CursorAndLength load(SelectFilter searchable) { + log.trace("Creating cursor for `{}`", searchable.getId()); + return new CursorAndLength(listAllValues(searchable), getTotal(searchable)); + } - }); + }); /** * For a {@link SelectFilter} collect all relevant {@link TrieSearch}. @@ -108,17 +116,16 @@ public final List> getSearchesFor(SelectFilter sear final List references = searchable.getSearchReferences(); if (log.isTraceEnabled()) { - log.trace("Got {} as searchables for {}", references.stream().map(Searchable::toString).collect(Collectors.toList()), searchable.getId()); + log.trace( + "Got {} as searchables for {}", + references.stream().map(Searchable::toString).collect(Collectors.toList()), + searchable.getId()); } - return references.stream() - .map(searchCache::get) - .filter(Objects::nonNull) - .toList(); + return references.stream().map(searchCache::get).filter(Objects::nonNull).toList(); } - private Cursor listAllValues(SelectFilter searchable) { /* Don't worry, I am as confused as you are! @@ -130,13 +137,13 @@ private Cursor listAllValues(SelectFilter searchable) { final List> searchList = getSearchesFor(searchable); - final Iterator searches = Iterators.concat(Iterators.transform(searchList.iterator(), TrieSearch::iterator)); - final Iterator iterators = - Iterators.concat( - // We are always leading with the empty value. - Iterators.singletonIterator(new FrontendValue("", searchConfig.getEmptyLabel())), - searches - ); + final Iterator searches = Iterators.concat( + Iterators.transform(searchList.iterator(), TrieSearch::iterator)); + final Iterator iterators = Iterators.concat( + // We are always leading with the empty value. + Iterators.singletonIterator(new FrontendValue("", searchConfig.getEmptyLabel())), + searches + ); // Use Set to accomplish distinct values final Set seen = new HashSet<>(); @@ -172,12 +179,15 @@ public synchronized void addSearches(Map> * prevents from adding new values. */ public void registerValues(Searchable searchable, Collection values) { - TrieSearch search = searchCache.computeIfAbsent(searchable, (ignored) -> searchConfig.createSearch(searchable)); + TrieSearch search = searchCache.computeIfAbsent( + searchable, + (ignored) -> searchConfig.createSearch(searchable)); synchronized (search) { values.stream() - .map(value -> new FrontendValue(value, value)) - .forEach(value -> search.addItem(value, SearchProcessor.extractKeywords(value))); + .map(value -> new FrontendValue(value, value)) + .forEach( + value -> search.addItem(value, SearchProcessor.extractKeywords(value))); } } @@ -224,11 +234,15 @@ public Job createUpdateFilterSearchJob(NamespaceStorage storage, Consumer managerSearchables, AtomicBoolean cancelledState, ProgressReporter progressReporter) throws InterruptedException { + public void indexManagerResidingSearches( + Set managerSearchables, + AtomicBoolean cancelledState, + ProgressReporter progressReporter) throws InterruptedException { // Most computations are cheap but data intensive: we fork here to use as many cores as possible. - try(final ExecutorService service = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() - 1)) { + try (final ExecutorService service = Executors.newFixedThreadPool( + Runtime.getRuntime().availableProcessors() - 1)) { final Map> searchCache = new ConcurrentHashMap<>(); for (Searchable searchable : managerSearchables) { @@ -248,13 +262,12 @@ public void indexManagerResidingSearches(Set managerSearchables, Ato searchCache.put(searchable, search); log.debug( - "DONE collecting {} entries for `{}`, within {}", - search.calculateSize(), - searchable, - watch + "DONE collecting {} entries for `{}`, within {}", + search.calculateSize(), + searchable, + watch ); - } - catch (Exception e) { + } catch (Exception e) { log.error("Failed to create search for {}", searchable, e); } @@ -299,7 +312,7 @@ public ConceptsProcessor.ExactFilterValueResult findExact(SelectFilter filter // search in the full text engine final Set openSearchTerms = new HashSet<>(searchTerms); - for (final Iterator iterator = openSearchTerms.iterator(); iterator.hasNext(); ) { + for (final Iterator iterator = openSearchTerms.iterator(); iterator.hasNext();) { final String searchTerm = iterator.next(); final List results = findExact(filter, searchTerm); @@ -316,7 +329,11 @@ public ConceptsProcessor.ExactFilterValueResult findExact(SelectFilter filter } @Override - public ConceptsProcessor.AutoCompleteResult query(SelectFilter filter, String maybeText, int itemsPerPage, int pageNumber) { + public ConceptsProcessor.AutoCompleteResult query( + SelectFilter filter, + String maybeText, + int itemsPerPage, + int pageNumber) { final int startIncl = itemsPerPage * pageNumber; final int endExcl = startIncl + itemsPerPage; @@ -336,10 +353,11 @@ public ConceptsProcessor.AutoCompleteResult query(SelectFilter filter, String return new ConceptsProcessor.AutoCompleteResult(Collections.emptyList(), fullResult.size()); } - return new ConceptsProcessor.AutoCompleteResult(fullResult.subList(startIncl, Math.min(fullResult.size(), endExcl)), fullResult.size()); - } - catch (ExecutionException e) { - log.warn("Failed to search for \"{}\".", maybeText, passExceptionOnTrace(log,e)); + return new ConceptsProcessor.AutoCompleteResult( + fullResult.subList(startIncl, Math.min(fullResult.size(), endExcl)), + fullResult.size()); + } catch (ExecutionException e) { + log.warn("Failed to search for \"{}\".", maybeText, passExceptionOnTrace(log, e)); return new ConceptsProcessor.AutoCompleteResult(Collections.emptyList(), 0); } } @@ -376,11 +394,9 @@ public void execute() { final Column column = columnId.resolve(); try { filterSearch.finalizeSearch(column); - } - catch (Exception e) { + } catch (Exception e) { log.warn("Unable to shrink search for {}", column, e); - } - finally { + } finally { getProgressReporter().report(1); } } @@ -393,11 +409,9 @@ public void execute() { try { final long total = filterSearch.getTotal(filter); log.trace("Filter '{}' totals: {}", filter, total); - } - catch (Exception e) { + } catch (Exception e) { log.warn("Unable to calculate totals for filter '{}'", filter.getId(), e); - } - finally { + } finally { getProgressReporter().report(1); } } diff --git a/backend/src/main/java/com/bakdata/conquery/util/search/internal/TrieSearch.java b/backend/src/main/java/com/bakdata/conquery/util/search/internal/TrieSearch.java index 363c51a42d1..1e21389c0ea 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/search/internal/TrieSearch.java +++ b/backend/src/main/java/com/bakdata/conquery/util/search/internal/TrieSearch.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.util.search.internal; +import jakarta.validation.constraints.Min; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -17,7 +18,6 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; -import jakarta.validation.constraints.Min; import com.bakdata.conquery.util.search.Search; import com.google.common.base.Strings; @@ -96,11 +96,13 @@ public static > List topItems(int limit, Object2LongM // Sort items according to their weight, then limit. // Note that sorting is in descending order, meaning higher-scores are better. return itemWeights.object2LongEntrySet() - .stream() - .sorted(Comparator.comparing(Object2LongMap.Entry::getLongValue, Comparator.reverseOrder())) - .limit(limit) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); + .stream() + .sorted( + Comparator.comparing(Object2LongMap.Entry::getLongValue, Comparator.reverseOrder())) + .limit(limit) + .map( + Map.Entry::getKey) + .collect(Collectors.toList()); } public Object2LongMap collectWeights(Collection queries) { @@ -128,11 +130,11 @@ public Object2LongMap collectWeights(Collection queries) { } // Collectors::toMap throws IllegalStateException if there are duplicate keys - final Map> ngramHits = ngramSplit(query) - .distinct() - .collect(Collectors.toMap( - Function.identity(), - ng -> ngrams.getOrDefault(ng, Collections.emptyList()) + final Map> ngramHits = ngramSplit(query).distinct() + .collect( + Collectors.toMap( + Function.identity(), + ng -> ngrams.getOrDefault(ng, Collections.emptyList()) )); updateWeights(query, ngramHits, itemWeights, false); @@ -152,15 +154,20 @@ private Stream split(String keyword) { } return splitPattern.splitAsStream(keyword.trim()) - .map(String::trim) - .filter(StringUtils::isNotBlank) - .map(String::toLowerCase); + .map(String::trim) + .filter(StringUtils::isNotBlank) + .map( + String::toLowerCase); } /** * Calculate and update weights for all queried items */ - private void updateWeights(String query, final Map> items, Object2LongMap itemWeights, boolean original) { + private void updateWeights( + String query, + final Map> items, + Object2LongMap itemWeights, + boolean original) { if (items == null) { return; } @@ -192,7 +199,8 @@ public Stream ngramSplit(String word) { } return IntStream.range(0, word.length() - ngramLength + 1) - .mapToObj(start -> word.substring(start, start + ngramLength)); + .mapToObj( + start -> word.substring(start, start + ngramLength)); } /** @@ -205,8 +213,7 @@ private long weightWord(String query, String itemWord, boolean original) { // We prefer same length words. if (query.length() == itemWord.length()) { weight = EXACT_MATCH_WEIGHT; - } - else { + } else { weight = BASE_WEIGHT; } @@ -220,13 +227,14 @@ private long weightWord(String query, String itemWord, boolean original) { public List findExact(String searchTerm, int limit) { return Stream.of(searchTerm) - .flatMap(this::split) - .map(whole::get) - .filter(Objects::nonNull) - .flatMap(List::stream) - .distinct() - .limit(limit) - .collect(Collectors.toList()); + .flatMap(this::split) + .map(whole::get) + .filter(Objects::nonNull) + .flatMap( + List::stream) + .distinct() + .limit(limit) + .collect(Collectors.toList()); } public void addItem(T item, List keywords) { @@ -237,9 +245,10 @@ public void addItem(T item, List keywords) { // Associate item with all extracted keywords keywords.stream() - .filter(Predicate.not(Strings::isNullOrEmpty)) - .flatMap(this::split) - .forEach(word -> doPut(word, item, barrier)); + .filter(Predicate.not(Strings::isNullOrEmpty)) + .flatMap(this::split) + .forEach( + word -> doPut(word, item, barrier)); } private void ensureWriteable() { @@ -252,9 +261,9 @@ private void ensureWriteable() { private void doPut(String word, T item, Set barrier) { whole.computeIfAbsent(word, (ignored) -> new ArrayList<>()).add(item); - ngramSplit(word) - .filter(barrier::add) - .forEach(key -> ngrams.computeIfAbsent(key, (ignored) -> new ArrayList<>()).add(item)); + ngramSplit(word).filter(barrier::add) + .forEach( + key -> ngrams.computeIfAbsent(key, (ignored) -> new ArrayList<>()).add(item)); } public boolean isWriteable() { @@ -293,9 +302,7 @@ public long calculateSize() { } public Stream stream() { - return whole.values().stream() - .flatMap(Collection::stream) - .distinct(); + return whole.values().stream().flatMap(Collection::stream).distinct(); } public Iterator iterator() { @@ -303,9 +310,9 @@ public Iterator iterator() { final Set seen = new HashSet<>(); return Iterators.filter( - Iterators.concat(Iterators.transform(whole.values().iterator(), Collection::iterator)), - seen::add + Iterators.concat(Iterators.transform(whole.values().iterator(), Collection::iterator)), + seen::add ); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/util/search/solr/ChunkSubmitter.java b/backend/src/main/java/com/bakdata/conquery/util/search/solr/ChunkSubmitter.java index 5fdbc61a029..1d12ab53573 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/search/solr/ChunkSubmitter.java +++ b/backend/src/main/java/com/bakdata/conquery/util/search/solr/ChunkSubmitter.java @@ -74,24 +74,28 @@ public void insertIntoChunk(SolrFrontendValue solrFrontendValue) { } private synchronized CompletableFuture submitChunk() { - submitter = submitter.thenRunAsync( - () -> { - if (openChunksQueue.isEmpty()) { - return; - } - - // We chunk here for performance. - // Too many small document request cause a lot of overhead. - // A too large chunk slows request submission and solr. - do { - int chunksCount = openChunksQueue.size(); - - Collection chunk = openChunksQueue.poll(); - log.debug("Adding {} (of currently ca. {}) documents for {}", chunk.size(), chunksCount*updateChunkSize, searchable); - registerValues(chunk); - } while (!openChunksQueue.isEmpty()); - }, - executor + submitter = submitter.thenRunAsync( + () -> { + if (openChunksQueue.isEmpty()) { + return; + } + + // We chunk here for performance. + // Too many small document request cause a lot of overhead. + // A too large chunk slows request submission and solr. + do { + int chunksCount = openChunksQueue.size(); + + Collection chunk = openChunksQueue.poll(); + log.debug( + "Adding {} (of currently ca. {}) documents for {}", + chunk.size(), + chunksCount * updateChunkSize, + searchable); + registerValues(chunk); + } while (!openChunksQueue.isEmpty()); + }, + executor ); return submitter; @@ -111,12 +115,20 @@ private void registerValues(Collection solrFrontendValues) { try { Stopwatch stopwatch = Stopwatch.createStarted(); - log.trace("BEGIN registering {} values to {} for {}", solrFrontendValues.size(), solrClient.getDefaultCollection(), searchable); + log.trace( + "BEGIN registering {} values to {} for {}", + solrFrontendValues.size(), + solrClient.getDefaultCollection(), + searchable); solrClient.addBeans(solrFrontendValues); // do not commit yet - log.trace("DONE registering {} values to {} for {} in {}", solrFrontendValues.size(), solrClient.getDefaultCollection(), searchable, stopwatch); - - } - catch (SolrServerException | IOException e) { + log.trace( + "DONE registering {} values to {} for {} in {}", + solrFrontendValues.size(), + solrClient.getDefaultCollection(), + searchable, + stopwatch); + + } catch (SolrServerException | IOException e) { clientError = e; try { solrClient.close(); diff --git a/backend/src/main/java/com/bakdata/conquery/util/search/solr/FilterValueIndexer.java b/backend/src/main/java/com/bakdata/conquery/util/search/solr/FilterValueIndexer.java index 24cf8a6bf9f..c30b7c2dff1 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/search/solr/FilterValueIndexer.java +++ b/backend/src/main/java/com/bakdata/conquery/util/search/solr/FilterValueIndexer.java @@ -70,8 +70,7 @@ public long calculateSize() { try { QueryResponse response = chunkSubmitter.getSolrClient().query(query); return response.getResults().getNumFound(); - } - catch (SolrServerException | IOException e) { + } catch (SolrServerException | IOException e) { throw new RuntimeException(e); } } @@ -100,7 +99,10 @@ private void insertIntoChunk(SolrFrontendValue solrFrontendValue) { if (solrFrontendValue.value_s.isEmpty()) { if (seenEmpty) { - log.trace("Skip indexing of {} for {}, because its 'value' is empty and was already added.", solrFrontendValue, getSearchable()); + log.trace( + "Skip indexing of {} for {}, because its 'value' is empty and was already added.", + solrFrontendValue, + getSearchable()); return; } seenEmpty = true; @@ -108,7 +110,10 @@ private void insertIntoChunk(SolrFrontendValue solrFrontendValue) { } if (!seenValues.add(solrFrontendValue.value_s)) { - log.trace("Skip indexing of {} for {}, because its 'value' has already been submitted to solr.", solrFrontendValue, getSearchable()); + log.trace( + "Skip indexing of {} for {}, because its 'value' has already been submitted to solr.", + solrFrontendValue, + getSearchable()); return; } @@ -126,7 +131,12 @@ public void registerValuesRaw(Collection values) { if (value == null || value.isBlank()) { continue; } - SolrFrontendValue solrFrontendValue = new SolrFrontendValue(getSearchable(), sourcePriority, value, null, null); + SolrFrontendValue solrFrontendValue = new SolrFrontendValue( + getSearchable(), + sourcePriority, + value, + null, + null); insertIntoChunk(solrFrontendValue); } } diff --git a/backend/src/main/java/com/bakdata/conquery/util/search/solr/FilterValueSearch.java b/backend/src/main/java/com/bakdata/conquery/util/search/solr/FilterValueSearch.java index 3160735400a..06e8cf5ad31 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/search/solr/FilterValueSearch.java +++ b/backend/src/main/java/com/bakdata/conquery/util/search/solr/FilterValueSearch.java @@ -1,5 +1,17 @@ package com.bakdata.conquery.util.search.solr; +import java.io.IOException; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.annotation.CheckForNull; +import javax.annotation.Nullable; + import com.bakdata.conquery.apiv1.frontend.FrontendValue; import com.bakdata.conquery.models.config.search.solr.FilterValueConfig; import com.bakdata.conquery.models.datasets.concepts.Searchable; @@ -24,18 +36,6 @@ import org.apache.solr.common.SolrDocument; import org.jetbrains.annotations.NotNull; -import javax.annotation.CheckForNull; -import javax.annotation.Nullable; -import java.io.IOException; -import java.net.URLDecoder; -import java.nio.charset.StandardCharsets; -import java.util.*; -import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.stream.Collectors; -import java.util.stream.Stream; - /** * Helper class to abstract/capsule the {@link Searchable}s of the filter away. * The data was/is imported to Solr by a {@link FilterValueIndexer} for each {@link Searchable}, because a single {@link Searchable} might be used by multiple {@link Filter}s. @@ -44,37 +44,37 @@ @Slf4j public class FilterValueSearch { - public static final int SOLR_MAX_URI_LENGTH = 5000 /* bytes */ ; // with some buffer, actual limit is 8192, but setting this to 6000 already causes failures + public static final int SOLR_MAX_URI_LENGTH = 5000 /* bytes */ ; // with some buffer, actual limit is 8192, but setting this to 6000 already causes failures private final SelectFilter filter; - private final SolrProcessor processor; - private final SolrClient solrClient; - private final FilterValueConfig filterValueConfig; - - public List getSearchesFor(SelectFilter searchable, boolean withEmptySource) { - List searchReferences = new ArrayList<>(searchable.getSearchReferences()); - - if (withEmptySource) { - // Patchup searchables - searchReferences.add(SolrEmptySeachable.INSTANCE); - } - - return searchReferences.stream().map(processor::getIndexerFor).toList(); - } - - - /** - * Creates a filter query (which is cached by solr) for the subset of documents originating from the searchables related to this query. - * - * @param withEmptySource Also include the special empty value source in the results, which allows in conquery to filter for empty fields. - * @return Query string that is a group of the searchable ids for the {@link FilterValueSearch#filter}. - */ - private @NotNull String buildFilterQuery(boolean withEmptySource) { - List indexers = getSearchesFor(filter, withEmptySource); - return indexers.stream() - .map(FilterValueIndexer::getSearchable) - // The name of the searchable was already escaped at the creation of SolrSearch - .collect(Collectors.joining(" ", "%s:(".formatted(SolrFrontendValue.Fields.searchable_s), ")")); - } + private final SolrProcessor processor; + private final SolrClient solrClient; + private final FilterValueConfig filterValueConfig; + + public List getSearchesFor(SelectFilter searchable, boolean withEmptySource) { + List searchReferences = new ArrayList<>(searchable.getSearchReferences()); + + if (withEmptySource) { + // Patchup searchables + searchReferences.add(SolrEmptySeachable.INSTANCE); + } + + return searchReferences.stream().map(processor::getIndexerFor).toList(); + } + + + /** + * Creates a filter query (which is cached by solr) for the subset of documents originating from the searchables related to this query. + * + * @param withEmptySource Also include the special empty value source in the results, which allows in conquery to filter for empty fields. + * @return Query string that is a group of the searchable ids for the {@link FilterValueSearch#filter}. + */ + private @NotNull String buildFilterQuery(boolean withEmptySource) { + List indexers = getSearchesFor(filter, withEmptySource); + return indexers.stream() + .map(FilterValueIndexer::getSearchable) + // The name of the searchable was already escaped at the creation of SolrSearch + .collect(Collectors.joining(" ", "%s:(".formatted(SolrFrontendValue.Fields.searchable_s), ")")); + } /** * Query syntax reference @@ -88,197 +88,234 @@ public AutoCompleteResult topItems(String text, Integer start, @Nullable Integer return sendQuery(text, start, limit, true, true); } text = Arrays.stream(text.split("\\s")) - // Skip blanks - .filter(Predicate.not(String::isBlank)) - // Escape - .map(ClientUtils::escapeQueryChars) - // Resolve Query template - .map((term) -> { - Map valuesMap = Map.of("term", term); - StringSubstitutor sub = new StringSubstitutor(valuesMap); - return sub.replace(filterValueConfig.getQueryTemplate()); - }) - .collect(Collectors.joining(" AND ")); + // Skip blanks + .filter(Predicate.not(String::isBlank)) + // Escape + .map(ClientUtils::escapeQueryChars) + // Resolve Query template + .map((term) -> { + Map valuesMap = Map.of("term", term); + StringSubstitutor sub = new StringSubstitutor(valuesMap); + return sub.replace(filterValueConfig.getQueryTemplate()); + }) + .collect(Collectors.joining(" AND ")); return sendQuery(text, start, limit, false, false); } - private @NotNull AutoCompleteResult sendQuery(String queryString, Integer start, @CheckForNull Integer limit, boolean withEmptySource, boolean sort) { - SolrQuery query = buildSolrQuery(queryString, start, limit, sort, withEmptySource, true); - - String decodedQuery = URLDecoder.decode(String.valueOf(query), StandardCharsets.UTF_8); - int queryHash = decodedQuery.hashCode(); - log.info("Query [{}] created: {}", queryHash, decodedQuery); - - try { - - List beans = new ArrayList<>(); - final AtomicLong numFound = new AtomicLong(); - final StreamingResponseCallback callback = new StreamingResponseCallback() { - @Override - public void streamSolrDocument(SolrDocument doc) { - SolrFrontendValue bean = solrClient.getBinder().getBean(SolrFrontendValue.class, doc); - beans.add(bean.toFrontendValue()); - } - - @Override - public void streamDocListInfo(long numFoundCallBack, long start, Float maxScore) { - numFound.set(numFoundCallBack); - } - }; - - final QueryRequest request = new QueryRequest(query, SolrRequest.METHOD.POST); - request.setResponseParser(new StreamingBinaryResponseParser(callback)); - final QueryResponse response = request.process(solrClient); - - log.debug("Query [{}] Found: {} | Collected: {} | QTime: {} | ElapsedTime: {}", queryHash, numFound.get(), beans.size(), response.getQTime(), response.getElapsedTime()); - - return new AutoCompleteResult(beans, numFound.get()); - - } catch (SolrServerException | IOException e) { - throw new RuntimeException(e); - } - } - - /** - * @param withEmptySource Includes the special empty source within the result set. Its single value is usually only needed for the default query. - * @param collapseSources Collapse documents on {@link SolrFrontendValue#value_s} by source priority. Different sources can hold documents to the same value. Then true the document with the highest priority is included. - */ - private @NotNull SolrQuery buildSolrQuery(String queryString, Integer start, @CheckForNull Integer limit, boolean sort, boolean withEmptySource, boolean collapseSources) { - String filterQuery = buildFilterQuery(withEmptySource); - - SolrQuery query = new SolrQuery(queryString); - query.addFilterQuery(filterQuery); - query.addField(SolrFrontendValue.Fields.value_s); - query.addField(SolrFrontendValue.Fields.label_t); - query.addField(SolrFrontendValue.Fields.optionValue_s); - query.setStart(start); - query.setRows(limit); - - if (sort) { - query.setSort(SolrQuery.SortClause.asc(SolrFrontendValue.Fields.sourcePriority_i)); - query.addSort(SolrQuery.SortClause.asc(filterValueConfig.getDefaultSearchSortField())); - } - - if (collapseSources) { - // Collapse the results with equal "value" field. Only the one with the highest score remains. - // This only works if solr is not sharded (or collapsing documents are on the same shard) - // We set 'nullPolicy=expand' so we do not suppress the empty label entry - query.addFilterQuery("{!collapse field=%s min=%s nullPolicy=expand}".formatted(SolrFrontendValue.Fields.value_s, SolrFrontendValue.Fields.sourcePriority_i)); - } - - return query; - } - - /** - * Find (almost) exact matches for the provided terms. Lowercased search terms are compared to labels and values of solr docs - */ - public ConceptsProcessor.ExactFilterValueResult exact(Collection terms) { - if (terms == null || terms.isEmpty()) { - return new ConceptsProcessor.ExactFilterValueResult(List.of(), terms); - } - - final List escapedTerms = terms.stream() - .filter(Predicate.not(String::isBlank)) - .distinct() - .map(ClientUtils::escapeQueryChars) - .toList(); - - // Build map of all values (normalized -> original) where we remove the one we found from. This leaves us with the unresolved values in the end - final Map unresolvedMap = terms.stream().collect(Collectors.toMap(String::toLowerCase, Function.identity(), (v1, v2) -> v1)); - final List resolved = new ArrayList<>(terms.size()); - - - // TODO also use Solr's POST method to avoid too long URIs (see sendQuery) - List> chunks = chunkByUriLength(escapedTerms); - int chunkIndex = 1; - final int chunkCount = chunks.size(); - int source_count = getSearchesFor(filter, false).size(); - for (List chunk : chunks) { + private @NotNull AutoCompleteResult sendQuery( + String queryString, + Integer start, + @CheckForNull Integer limit, + boolean withEmptySource, + boolean sort) { + SolrQuery query = buildSolrQuery(queryString, start, limit, sort, withEmptySource, true); + + String decodedQuery = URLDecoder.decode(String.valueOf(query), StandardCharsets.UTF_8); + int queryHash = decodedQuery.hashCode(); + log.info("Query [{}] created: {}", queryHash, decodedQuery); + + try { + + List beans = new ArrayList<>(); + final AtomicLong numFound = new AtomicLong(); + final StreamingResponseCallback callback = new StreamingResponseCallback() { + @Override + public void streamSolrDocument(SolrDocument doc) { + SolrFrontendValue bean = solrClient.getBinder().getBean(SolrFrontendValue.class, doc); + beans.add(bean.toFrontendValue()); + } + + @Override + public void streamDocListInfo(long numFoundCallBack, long start, Float maxScore) { + numFound.set(numFoundCallBack); + } + }; + + final QueryRequest request = new QueryRequest(query, SolrRequest.METHOD.POST); + request.setResponseParser(new StreamingBinaryResponseParser(callback)); + final QueryResponse response = request.process(solrClient); + + log.debug( + "Query [{}] Found: {} | Collected: {} | QTime: {} | ElapsedTime: {}", + queryHash, + numFound.get(), + beans.size(), + response.getQTime(), + response.getElapsedTime()); + + return new AutoCompleteResult(beans, numFound.get()); + + } catch (SolrServerException | IOException e) { + throw new RuntimeException(e); + } + } - String queryString = buildExactQuery(chunk); + /** + * @param withEmptySource Includes the special empty source within the result set. Its single value is usually only needed for the default query. + * @param collapseSources Collapse documents on {@link SolrFrontendValue#value_s} by source priority. Different sources can hold documents to the same value. Then true the document with the highest priority is included. + */ + private @NotNull SolrQuery buildSolrQuery( + String queryString, + Integer start, + @CheckForNull Integer limit, + boolean sort, + boolean withEmptySource, + boolean collapseSources) { + String filterQuery = buildFilterQuery(withEmptySource); + + SolrQuery query = new SolrQuery(queryString); + query.addFilterQuery(filterQuery); + query.addField(SolrFrontendValue.Fields.value_s); + query.addField(SolrFrontendValue.Fields.label_t); + query.addField(SolrFrontendValue.Fields.optionValue_s); + query.setStart(start); + query.setRows(limit); + + if (sort) { + query.setSort(SolrQuery.SortClause.asc(SolrFrontendValue.Fields.sourcePriority_i)); + query.addSort(SolrQuery.SortClause.asc(filterValueConfig.getDefaultSearchSortField())); + } - if (StringUtils.isBlank(queryString)) { - return new ConceptsProcessor.ExactFilterValueResult(List.of(), terms); - } + if (collapseSources) { + // Collapse the results with equal "value" field. Only the one with the highest score remains. + // This only works if solr is not sharded (or collapsing documents are on the same shard) + // We set 'nullPolicy=expand' so we do not suppress the empty label entry + query.addFilterQuery( + "{!collapse field=%s min=%s nullPolicy=expand}".formatted( + SolrFrontendValue.Fields.value_s, + SolrFrontendValue.Fields.sourcePriority_i)); + } - // The batchsize is twice the size of the chunk size because a term is often (at most) found in two documents (from the column and from a mapping) - final int batchSize = chunk.size() * source_count; - - final AtomicLong numFound = new AtomicLong(); - try { - List resolvedValues = new ArrayList<>(); - - // We sort to return value with the highest source priority and get the best description - SolrQuery solrQuery = buildSolrQuery(queryString, 0, batchSize, true, false, false); - - - String decodedQuery = URLDecoder.decode(String.valueOf(solrQuery), StandardCharsets.UTF_8); - int queryHash = decodedQuery.hashCode(); - log.trace("Query [{}] ({}/{}) created: {}", queryHash, chunkIndex, chunkCount, decodedQuery); - - int queryByteLength = solrQuery.toString().getBytes(StandardCharsets.UTF_8).length; - log.trace("Query [{}] length in bytes: {}", queryHash, queryByteLength); + return query; + } - QueryResponse response = solrClient.queryAndStreamResponse(solrQuery, new StreamingResponseCallback() { - @Override - public void streamSolrDocument(SolrDocument doc) { - log.trace("Query [{}] received document: {}", queryHash, doc); - if (unresolvedMap.isEmpty()) { - // Shortcut: everything was resolved - return; - } + /** + * Find (almost) exact matches for the provided terms. Lowercased search terms are compared to labels and values of solr docs + */ + public ConceptsProcessor.ExactFilterValueResult exact(Collection terms) { + if (terms == null || terms.isEmpty()) { + return new ConceptsProcessor.ExactFilterValueResult(List.of(), terms); + } - SolrFrontendValue bean = solrClient.getBinder().getBean(SolrFrontendValue.class, doc); + final List escapedTerms = terms.stream() + .filter(Predicate.not(String::isBlank)) + .distinct() + .map( + ClientUtils::escapeQueryChars) + .toList(); + // Build map of all values (normalized -> original) where we remove the one we found from. This leaves us with the unresolved values in the end + final Map unresolvedMap = terms.stream() + .collect( + Collectors.toMap(String::toLowerCase, Function.identity(), (v1, v2) -> v1)); + final List resolved = new ArrayList<>(terms.size()); - // Remove from unresolved and add to resolved values if either value or label matches - if (unresolvedMap.remove(bean.value_s.toLowerCase()) != null || (bean.label_t != null && unresolvedMap.remove(bean.label_t.toLowerCase()) != null)) { - FrontendValue frontendValue = bean.toFrontendValue(); - resolvedValues.add(frontendValue); - - } + // TODO also use Solr's POST method to avoid too long URIs (see sendQuery) + List> chunks = chunkByUriLength(escapedTerms); + int chunkIndex = 1; + final int chunkCount = chunks.size(); + int source_count = getSearchesFor(filter, false).size(); + for (List chunk : chunks) { - } - - @Override - public void streamDocListInfo(long numFoundCallBack, long start, Float maxScore) { - numFound.set(numFoundCallBack); - if (numFoundCallBack > batchSize) { - log.warn("Query found more documents ({}) than expected ({}). We expect a term to be found in at most {} documents (from a column and a mapping).", numFoundCallBack, batchSize, source_count); - } - } - }); - log.trace("Query [{}] ({}/{}) Found: {} | Collected: {} | QTime: {} | ElapsedTime: {}", queryHash, chunkIndex, chunkCount, numFound.get(), resolvedValues.size(), response.getQTime(), response.getElapsedTime()); + String queryString = buildExactQuery(chunk); - chunkIndex++; - resolved.addAll(resolvedValues); + if (StringUtils.isBlank(queryString)) { + return new ConceptsProcessor.ExactFilterValueResult(List.of(), terms); + } - } catch (SolrServerException | IOException e) { - throw new RuntimeException(e); - } - } + // The batchsize is twice the size of the chunk size because a term is often (at most) found in two documents (from the column and from a mapping) + final int batchSize = chunk.size() * source_count; + + final AtomicLong numFound = new AtomicLong(); + try { + List resolvedValues = new ArrayList<>(); + + // We sort to return value with the highest source priority and get the best description + SolrQuery solrQuery = buildSolrQuery(queryString, 0, batchSize, true, false, false); + + + String decodedQuery = URLDecoder.decode(String.valueOf(solrQuery), StandardCharsets.UTF_8); + int queryHash = decodedQuery.hashCode(); + log.trace("Query [{}] ({}/{}) created: {}", queryHash, chunkIndex, chunkCount, decodedQuery); + + int queryByteLength = solrQuery.toString().getBytes(StandardCharsets.UTF_8).length; + log.trace("Query [{}] length in bytes: {}", queryHash, queryByteLength); + + QueryResponse response = solrClient.queryAndStreamResponse(solrQuery, new StreamingResponseCallback() { + @Override + public void streamSolrDocument(SolrDocument doc) { + log.trace("Query [{}] received document: {}", queryHash, doc); + if (unresolvedMap.isEmpty()) { + // Shortcut: everything was resolved + return; + } + + SolrFrontendValue bean = solrClient.getBinder().getBean(SolrFrontendValue.class, doc); + + + // Remove from unresolved and add to resolved values if either value or label matches + if (unresolvedMap.remove( + bean.value_s.toLowerCase()) != null || (bean.label_t != null && unresolvedMap.remove( + bean.label_t.toLowerCase()) != null)) { + + FrontendValue frontendValue = bean.toFrontendValue(); + resolvedValues.add(frontendValue); + + } + + } + + @Override + public void streamDocListInfo(long numFoundCallBack, long start, Float maxScore) { + numFound.set(numFoundCallBack); + if (numFoundCallBack > batchSize) { + log.warn( + "Query found more documents ({}) than expected ({}). We expect a term to be found in at most {} documents (from a column and a mapping).", + numFoundCallBack, + batchSize, + source_count); + } + } + }); + log.trace( + "Query [{}] ({}/{}) Found: {} | Collected: {} | QTime: {} | ElapsedTime: {}", + queryHash, + chunkIndex, + chunkCount, + numFound.get(), + resolvedValues.size(), + response.getQTime(), + response.getElapsedTime()); + + chunkIndex++; + resolved.addAll(resolvedValues); + + } catch (SolrServerException | IOException e) { + throw new RuntimeException(e); + } + } - return new ConceptsProcessor.ExactFilterValueResult(resolved, unresolvedMap.values()); - } + return new ConceptsProcessor.ExactFilterValueResult(resolved, unresolvedMap.values()); + } - private List> chunkByUriLength(Collection terms) { - List> result = new ArrayList<>(); - List current = new ArrayList<>(); + private List> chunkByUriLength(Collection terms) { + List> result = new ArrayList<>(); + List current = new ArrayList<>(); - for (String term : terms) { - if (StringUtils.isBlank(term)) { - continue; - } - current.add(term); + for (String term : terms) { + if (StringUtils.isBlank(term)) { + continue; + } + current.add(term); - String query = buildExactQuery(current); + String query = buildExactQuery(current); - int size = query.getBytes(StandardCharsets.UTF_8).length; + int size = query.getBytes(StandardCharsets.UTF_8).length; if (size <= SOLR_MAX_URI_LENGTH) { - // Length still fine, let's move on + // Length still fine, let's move on continue; } @@ -295,31 +332,31 @@ private List> chunkByUriLength(Collection terms) { } } - if (!current.isEmpty()) { - result.add(current); - } + if (!current.isEmpty()) { + result.add(current); + } - return result; - } + return result; + } - @NotNull - private static String buildExactQuery(List chunk) { - String finalTerms = chunk.stream().collect(Collectors.joining(" ", "(", ")")); + @NotNull + private static String buildExactQuery(List chunk) { + String finalTerms = chunk.stream().collect(Collectors.joining(" ", "(", ")")); - if (StringUtils.isBlank(finalTerms)) { - return finalTerms; - } + if (StringUtils.isBlank(finalTerms)) { + return finalTerms; + } - // We are matching on label and value. - // So if for reason a value is present in multiple sources (map, template, ...) but has different labels - // both can be found. + // We are matching on label and value. + // So if for reason a value is present in multiple sources (map, template, ...) but has different labels + // both can be found. return Stream.of( - SolrFrontendValue.Fields.value_s, - SolrFrontendValue.Fields.label_t - ) - .map(field -> "%s:%s".formatted(field, finalTerms)) - // We are not interested in the result score, so we make it static: ^=1 - .collect(Collectors.joining(" OR ", "(", ")^=1")); - } + SolrFrontendValue.Fields.value_s, + SolrFrontendValue.Fields.label_t + ) + .map(field -> "%s:%s".formatted(field, finalTerms)) + // We are not interested in the result score, so we make it static: ^=1 + .collect(Collectors.joining(" OR ", "(", ")^=1")); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/util/search/solr/SolrEmptySeachable.java b/backend/src/main/java/com/bakdata/conquery/util/search/solr/SolrEmptySeachable.java index 9ab4f06c17b..9632173d4ed 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/search/solr/SolrEmptySeachable.java +++ b/backend/src/main/java/com/bakdata/conquery/util/search/solr/SolrEmptySeachable.java @@ -7,26 +7,26 @@ @NoArgsConstructor(access = AccessLevel.PRIVATE) public class SolrEmptySeachable implements Searchable { - public static final SolrEmptySeachable INSTANCE = new SolrEmptySeachable(); + public static final SolrEmptySeachable INSTANCE = new SolrEmptySeachable(); - @Override - public String getSearchHandle() { - return "empty"; - } + @Override + public String getSearchHandle() { + return "empty"; + } - @Override - public int getMinSuffixLength() { - return 0; - } + @Override + public int getMinSuffixLength() { + return 0; + } - @Override - public boolean isGenerateSuffixes() { - return false; - } + @Override + public boolean isGenerateSuffixes() { + return false; + } - @Override - public boolean isSearchDisabled() { - return false; - } + @Override + public boolean isSearchDisabled() { + return false; + } } diff --git a/backend/src/main/java/com/bakdata/conquery/util/search/solr/SolrProcessor.java b/backend/src/main/java/com/bakdata/conquery/util/search/solr/SolrProcessor.java index 639e9c2edd7..4529c116b5e 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/search/solr/SolrProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/util/search/solr/SolrProcessor.java @@ -78,10 +78,8 @@ public class SolrProcessor implements SearchProcessor, Managed { * This is mainly used to decouple mina threads from the solr client in order to prevent blocking and to convert between {@link com.bakdata.conquery.models.messages.namespaces.specific.RegisterColumnValues}'s * and solr chunk sizes. */ - private final ExecutorService chunkDecoupleExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder() - .setNameFormat("solr-submitter-%d") - .setDaemon(true) - .build()); + private final ExecutorService chunkDecoupleExecutor = Executors.newSingleThreadExecutor( + new ThreadFactoryBuilder().setNameFormat("solr-submitter-%d").setDaemon(true).build()); private SolrClient solrSearchClient; private SolrClient solrIndexClient; @@ -114,16 +112,14 @@ private synchronized void refreshClients() { if (solrSearchClient != null) { try { solrSearchClient.close(); - } - catch (Exception e) { + } catch (Exception e) { log.warn("Failed to close solr search client", e); } } if (solrIndexClient != null) { try { solrIndexClient.close(); - } - catch (Exception e) { + } catch (Exception e) { log.warn("Failed to close solr index client", e); } } @@ -148,8 +144,7 @@ public void clearSearch() { try (SolrClient solrClient = solrSearchClientFactory.get()) { log.info("Clearing collection: {}", solrClient.getDefaultCollection()); solrClient.deleteByQuery("*:*"); - } - catch (SolrServerException | IOException e) { + } catch (SolrServerException | IOException e) { throw new RuntimeException(e); } } @@ -177,9 +172,7 @@ public void registerValues(Searchable searchable, Collection values) { */ private String buildNameForSearchable(Searchable searchable) { - String name = searchable instanceof Column column - ? getNameFromColumn(column) - : searchable.getSearchHandle(); + String name = searchable instanceof Column column ? getNameFromColumn(column) : searchable.getSearchHandle(); name = ClientUtils.escapeQueryChars(name); @@ -199,14 +192,17 @@ private String getNameFromColumn(Column column) { String nameForSearchable = buildNameForSearchable(searchable); int sourcePriority = getFilterValueSourcePriority(searchable); - return indexers.computeIfAbsent(nameForSearchable, - searchRef -> new FilterValueIndexer(sourcePriority, - new ChunkSubmitter(nameForSearchable, - solrIndexClient, - filterValueConfig.getUpdateChunkSize(), - chunkDecoupleExecutor - ) - ) + return indexers.computeIfAbsent( + nameForSearchable, + searchRef -> new FilterValueIndexer( + sourcePriority, + new ChunkSubmitter( + nameForSearchable, + solrIndexClient, + filterValueConfig.getUpdateChunkSize(), + chunkDecoupleExecutor + ) + ) ); } @@ -231,8 +227,10 @@ public AutoCompleteResult topItems(SelectFilter filter, String text, Integer } @Override - public void indexManagerResidingSearches(Set managerSearchables, AtomicBoolean cancelledState, ProgressReporter progressReporter) - throws InterruptedException { + public void indexManagerResidingSearches( + Set managerSearchables, + AtomicBoolean cancelledState, + ProgressReporter progressReporter) throws InterruptedException { // Index an empty result for all searches indexEmptyLabel(); @@ -240,7 +238,8 @@ public void indexManagerResidingSearches(Set managerSearchables, Ato progressReporter.setMax(managerSearchables.size()); progressReporter.start(); // Most computations are cheap but data intensive: we fork here to use as many cores as possible. - try (final ExecutorService service = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() - 1)) { + try (final ExecutorService service = Executors.newFixedThreadPool( + Runtime.getRuntime().availableProcessors() - 1)) { final Map> searchCache = new ConcurrentHashMap<>(); for (Searchable searchable : managerSearchables) { @@ -261,7 +260,10 @@ public void indexManagerResidingSearches(Set managerSearchables, Ato case FilterTemplate temp -> indexFilterTemplate(search, temp); case LabelMap labelMap -> indexLabelMap(search, labelMap); default -> { - log.error("Unsupported searchable of type {}, skipping: {}", searchable.getClass(), searchable); + log.error( + "Unsupported searchable of type {}, skipping: {}", + searchable.getClass(), + searchable); return; } } @@ -269,15 +271,13 @@ public void indexManagerResidingSearches(Set managerSearchables, Ato searchCache.put(searchable, search); log.debug( - "DONE collecting entries for `{}`, within {}", - searchable, - watch + "DONE collecting entries for `{}`, within {}", + searchable, + watch ); - } - catch (Exception e) { + } catch (Exception e) { log.error("Failed to create search for {}", searchable, e); - } - finally { + } finally { progressReporter.report(1); } @@ -312,12 +312,17 @@ private void indexLabelMap(FilterValueIndexer search, LabelMap labelMap) { BiMap delegate = labelMap.getDelegate(); FilterId id = labelMap.getId(); - final List collected = delegate.entrySet().stream() - .map(entry -> new FrontendValue(entry.getKey(), entry.getValue())) - .toList(); + final List collected = delegate.entrySet() + .stream() + .map( + entry -> new FrontendValue(entry.getKey(), entry.getValue())) + .toList(); if (log.isTraceEnabled()) { - log.trace("Labels for {}: `{}`", id, collected.stream().map(FrontendValue::toString).collect(Collectors.toList())); + log.trace( + "Labels for {}: `{}`", + id, + collected.stream().map(FrontendValue::toString).collect(Collectors.toList())); } StopWatch timer = StopWatch.createStarted(); @@ -345,15 +350,16 @@ private void indexFilterTemplate(FilterValueIndexer search, FilterTemplate temp) log.trace("Resolved filter template reference url for search '{}': {}", temp.getId(), resolvedURI); try { - final FrontendValueIndex ignore = temp.getIndexService().getIndex(new FrontendValueIndexKey( - resolvedURI, - temp.getColumnValue(), - temp.getValue(), - temp.getOptionValue(), - () -> search - )); - } - catch (IndexCreationException e) { + final FrontendValueIndex ignore = temp.getIndexService() + .getIndex( + new FrontendValueIndexKey( + resolvedURI, + temp.getColumnValue(), + temp.getValue(), + temp.getOptionValue(), + () -> search + )); + } catch (IndexCreationException e) { throw new RuntimeException(e); } } @@ -386,7 +392,10 @@ public void explicitCommit() throws SolrServerException, IOException { solrIndexClient.commit(); // Depending on the collection size optimize takes a long time, longer than our client timeout, so we don't wait here. solrIndexClient.optimize(false, false); - log.info("DONE explicit commit to core/collection (optimize might still be pending) {} in {}", solrIndexClient.getDefaultCollection(), stopwatch); + log.info( + "DONE explicit commit to core/collection (optimize might still be pending) {} in {}", + solrIndexClient.getDefaultCollection(), + stopwatch); } @@ -412,10 +421,15 @@ public void execute() throws Exception { } solrProcessor.explicitCommit(); getProgressReporter().report(1); - log.info("Finished commit on collection {} in {}", solrProcessor.solrIndexClient.getDefaultCollection(), stopwatch); - } - catch (Exception e) { - log.error("Unable to issue explicit commit on collection {}", solrProcessor.solrIndexClient.getDefaultCollection(), e); + log.info( + "Finished commit on collection {} in {}", + solrProcessor.solrIndexClient.getDefaultCollection(), + stopwatch); + } catch (Exception e) { + log.error( + "Unable to issue explicit commit on collection {}", + solrProcessor.solrIndexClient.getDefaultCollection(), + e); } } diff --git a/backend/src/main/java/com/bakdata/conquery/util/search/solr/entities/SolrFrontendValue.java b/backend/src/main/java/com/bakdata/conquery/util/search/solr/entities/SolrFrontendValue.java index 7b2349baae6..193e9bb3a5b 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/search/solr/entities/SolrFrontendValue.java +++ b/backend/src/main/java/com/bakdata/conquery/util/search/solr/entities/SolrFrontendValue.java @@ -68,7 +68,12 @@ public SolrFrontendValue(String searchable, int sourcePriority, FrontendValue fe } - public SolrFrontendValue(String searchable, int sourcePriority, @NonNull String value, String label, String optionValue) { + public SolrFrontendValue( + String searchable, + int sourcePriority, + @NonNull String value, + String label, + String optionValue) { this.id = buildId(searchable, value); this.searchable_s = searchable; this.sourcePriority_i = sourcePriority; @@ -79,7 +84,10 @@ public SolrFrontendValue(String searchable, int sourcePriority, @NonNull String } public FrontendValue toFrontendValue() { - return new FrontendValue(Objects.requireNonNullElse(value_s, ""), Objects.requireNonNullElse(label_t, value_s), optionValue_s); + return new FrontendValue( + Objects.requireNonNullElse(value_s, ""), + Objects.requireNonNullElse(label_t, value_s), + optionValue_s); } private static String buildId(String searchable, String value) { diff --git a/backend/src/main/java/com/bakdata/conquery/util/validation/CaffeineSpecValidator.java b/backend/src/main/java/com/bakdata/conquery/util/validation/CaffeineSpecValidator.java index fac2643f6fb..44a38dbc516 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/validation/CaffeineSpecValidator.java +++ b/backend/src/main/java/com/bakdata/conquery/util/validation/CaffeineSpecValidator.java @@ -19,17 +19,18 @@ public boolean isValid(String value, ConstraintValidatorContext context) { boolean isValid = true; try { CaffeineSpec.parse(value); - } - catch (Exception e) { + } catch (Exception e) { isValid = false; - context.buildConstraintViolationWithTemplate("CaffeineSpec is invalid:" + e.getMessage()).addConstraintViolation(); + context.buildConstraintViolationWithTemplate( + "CaffeineSpec is invalid:" + e.getMessage()).addConstraintViolation(); } if (softValue) { // As long as we work with concrete objects (not Proxies) we need to ensure that we always hand out the one already referenced value if possible - if(!value.contains("softValues")) { + if (!value.contains("softValues")) { isValid = false; - context.buildConstraintViolationWithTemplate("Currently softValues need to be configured").addConstraintViolation(); + context.buildConstraintViolationWithTemplate( + "Currently softValues need to be configured").addConstraintViolation(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/util/validation/ManualURI.java b/backend/src/main/java/com/bakdata/conquery/util/validation/ManualURI.java index 3b3eed1ef6b..f94a1055afd 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/validation/ManualURI.java +++ b/backend/src/main/java/com/bakdata/conquery/util/validation/ManualURI.java @@ -3,13 +3,12 @@ import static java.lang.annotation.ElementType.*; import static java.lang.annotation.RetentionPolicy.RUNTIME; +import jakarta.validation.Constraint; +import jakarta.validation.Payload; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.Target; -import jakarta.validation.Constraint; -import jakarta.validation.Payload; - @Target({ANNOTATION_TYPE, FIELD, TYPE_USE}) @Retention(RUNTIME) @Constraint(validatedBy = ManualURIValidator.class) @@ -19,5 +18,6 @@ Class[] groups() default {}; - @SuppressWarnings("UnusedDeclaration") Class[] payload() default {}; + @SuppressWarnings("UnusedDeclaration") + Class[] payload() default {}; } diff --git a/backend/src/main/java/com/bakdata/conquery/util/validation/ManualURIValidator.java b/backend/src/main/java/com/bakdata/conquery/util/validation/ManualURIValidator.java index 49cc6067f54..87bc1920e2d 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/validation/ManualURIValidator.java +++ b/backend/src/main/java/com/bakdata/conquery/util/validation/ManualURIValidator.java @@ -1,12 +1,11 @@ package com.bakdata.conquery.util.validation; +import jakarta.validation.ConstraintValidator; +import jakarta.validation.ConstraintValidatorContext; import java.net.MalformedURLException; import java.net.URI; import java.util.List; -import jakarta.validation.ConstraintValidator; -import jakarta.validation.ConstraintValidatorContext; - public class ManualURIValidator implements ConstraintValidator { @Override @@ -22,30 +21,27 @@ public boolean isValid(URI value, ConstraintValidatorContext context) { if (value.isOpaque()) { isValid = false; - context.buildConstraintViolationWithTemplate("The URI was opaque") - .addConstraintViolation(); + context.buildConstraintViolationWithTemplate("The URI was opaque").addConstraintViolation(); } if (value.isAbsolute()) { try { var unused = value.toURL(); - } - catch (MalformedURLException e) { + } catch (MalformedURLException e) { isValid = false; - context.buildConstraintViolationWithTemplate("Absolute URI is not a valid URL (" + e.getMessage() + ")") - .addConstraintViolation(); + context.buildConstraintViolationWithTemplate( + "Absolute URI is not a valid URL (" + e.getMessage() + ")").addConstraintViolation(); } if (!List.of("http", "https").contains(value.getScheme())) { isValid = false; - context.buildConstraintViolationWithTemplate("The URI has an unsupported Scheme: " + value.getScheme()) - .addConstraintViolation(); + context.buildConstraintViolationWithTemplate( + "The URI has an unsupported Scheme: " + value.getScheme()).addConstraintViolation(); } - } - else { + } else { if (value.getAuthority() != null) { isValid = false; - context.buildConstraintViolationWithTemplate("The URI is not absolute but has an authority: " + value.getAuthority()) - .addConstraintViolation(); + context.buildConstraintViolationWithTemplate( + "The URI is not absolute but has an authority: " + value.getAuthority()).addConstraintViolation(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/util/validation/SqlTableValidator.java b/backend/src/main/java/com/bakdata/conquery/util/validation/SqlTableValidator.java index 66167022298..d8382c3496c 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/validation/SqlTableValidator.java +++ b/backend/src/main/java/com/bakdata/conquery/util/validation/SqlTableValidator.java @@ -1,9 +1,9 @@ package com.bakdata.conquery.util.validation; +import jakarta.validation.ConstraintValidatorContext; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import jakarta.validation.ConstraintValidatorContext; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.Table; @@ -35,17 +35,15 @@ public boolean isValid(Table value, ConstraintValidatorContext context) { final Result result; try { // We don't use DSL.meta here because that can be excessively slow. - result = dslContext.select(DSL.asterisk()) - .from(DSL.name(value.getName())) - .limit(0) - .fetch(); - } - catch (DataAccessException e) { + result = dslContext.select(DSL.asterisk()).from(DSL.name(value.getName())).limit(0).fetch(); + } catch (DataAccessException e) { log.trace("Failed to test for table {}", value.getName(), e); - context.buildConstraintViolationWithTemplate("SQL table %s does not exist".formatted(value.getName())) - .addPropertyNode("name") - .addConstraintViolation(); + context.buildConstraintViolationWithTemplate( + "SQL table %s does not exist".formatted(value.getName())) + .addPropertyNode( + "name") + .addConstraintViolation(); return false; } @@ -66,22 +64,21 @@ public boolean isValid(Table value, ConstraintValidatorContext context) { if (field == null) { // NOTE: The Path to the property is not factually correct, but the error messages are much more readable that way. - context.buildConstraintViolationWithTemplate("SQL Column `%s.%s` does not exist".formatted(value.getName(), column.getName())) - .addPropertyNode("columns") - .addPropertyNode(column.getName()) - .addConstraintViolation(); + context.buildConstraintViolationWithTemplate( + "SQL Column `%s.%s` does not exist".formatted(value.getName(), column.getName())) + .addPropertyNode( + "columns") + .addPropertyNode(column.getName()) + .addConstraintViolation(); valid = false; - } - else if (!dialect.isTypeCompatible(field, column.getType())) { - context.buildConstraintViolationWithTemplate("SQL Column `%s %s.%s` does not match required type %s".formatted( - field.getDataType(), - value.getName(), - column.getName(), - column.getType().name() - )) - .addPropertyNode("columns") - .addPropertyNode(column.getName()) - .addConstraintViolation(); + } else if (!dialect.isTypeCompatible(field, column.getType())) { + context.buildConstraintViolationWithTemplate( + "SQL Column `%s %s.%s` does not match required type %s".formatted( + field.getDataType(), + value.getName(), + column.getName(), + column.getType().name() + )).addPropertyNode("columns").addPropertyNode(column.getName()).addConstraintViolation(); valid = false; } } diff --git a/backend/src/main/java/com/bakdata/conquery/util/validation/ValidCaffeineSpec.java b/backend/src/main/java/com/bakdata/conquery/util/validation/ValidCaffeineSpec.java index 3c5d9eed820..e633b672f6b 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/validation/ValidCaffeineSpec.java +++ b/backend/src/main/java/com/bakdata/conquery/util/validation/ValidCaffeineSpec.java @@ -3,11 +3,11 @@ import static java.lang.annotation.ElementType.*; import static java.lang.annotation.RetentionPolicy.RUNTIME; +import jakarta.validation.Constraint; +import jakarta.validation.Payload; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.Target; -import jakarta.validation.Constraint; -import jakarta.validation.Payload; @Target({ANNOTATION_TYPE, FIELD, TYPE_USE}) @Retention(RUNTIME) @@ -18,7 +18,8 @@ Class[] groups() default {}; - @SuppressWarnings("UnusedDeclaration") Class[] payload() default {}; + @SuppressWarnings("UnusedDeclaration") + Class[] payload() default {}; /** * SoftValues must be configured in the spec. diff --git a/backend/src/main/java/com/bakdata/conquery/util/validation/ValidConqueryPermission.java b/backend/src/main/java/com/bakdata/conquery/util/validation/ValidConqueryPermission.java index d91c34567d2..279ddec51ed 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/validation/ValidConqueryPermission.java +++ b/backend/src/main/java/com/bakdata/conquery/util/validation/ValidConqueryPermission.java @@ -4,7 +4,6 @@ import jakarta.validation.Payload; import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.Pattern; - import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -14,13 +13,18 @@ * Validate Shiro permissions */ @NotBlank -@Pattern(regexp=ValidConqueryPermission.PERMISSION_PATTERN, message="Provide a valid shiro permission string with 3 parts at most, following this pattern: " + ValidConqueryPermission.PERMISSION_PATTERN) +@Pattern( + regexp = ValidConqueryPermission.PERMISSION_PATTERN, + message = "Provide a valid shiro permission string with 3 parts at most, following this pattern: " + ValidConqueryPermission.PERMISSION_PATTERN) @Target({ElementType.FIELD, ElementType.PARAMETER}) @Retention(RetentionPolicy.RUNTIME) @Constraint(validatedBy = {}) // No custom validator needed if only using built-ins public @interface ValidConqueryPermission { String PERMISSION_PATTERN = "^[\\w,*-]+(?::[\\w,*@-]+){0,2}$"; + String message() default "Invalid permission"; + Class[] groups() default {}; + Class[] payload() default {}; } diff --git a/backend/src/main/java/com/bakdata/conquery/util/validation/ValidSqlTable.java b/backend/src/main/java/com/bakdata/conquery/util/validation/ValidSqlTable.java index 931e428c97e..ba79653e281 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/validation/ValidSqlTable.java +++ b/backend/src/main/java/com/bakdata/conquery/util/validation/ValidSqlTable.java @@ -3,11 +3,11 @@ import static java.lang.annotation.ElementType.*; import static java.lang.annotation.RetentionPolicy.RUNTIME; +import jakarta.validation.Constraint; +import jakarta.validation.Payload; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.Target; -import jakarta.validation.Constraint; -import jakarta.validation.Payload; @Target({ANNOTATION_TYPE, FIELD, TYPE_USE}) @Retention(RUNTIME) @@ -18,6 +18,7 @@ Class[] groups() default {}; - @SuppressWarnings("UnusedDeclaration") Class[] payload() default {}; + @SuppressWarnings("UnusedDeclaration") + Class[] payload() default {}; } diff --git a/backend/src/main/java/org/apache/hadoop/package-info.java b/backend/src/main/java/org/apache/hadoop/package-info.java index 4b4e2e24c73..9459254a0da 100644 --- a/backend/src/main/java/org/apache/hadoop/package-info.java +++ b/backend/src/main/java/org/apache/hadoop/package-info.java @@ -10,4 +10,4 @@ *

* This saves us packaging a huge dependency. */ -package org.apache.hadoop; \ No newline at end of file +package org.apache.hadoop; diff --git a/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java b/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java index e9a9a694c8f..69bf9754e29 100644 --- a/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java +++ b/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java @@ -1,339 +1,390 @@ -package com.bakdata.conquery.api; - -import static com.bakdata.conquery.models.execution.ExecutionState.*; -import static org.assertj.core.api.Assertions.assertThat; - -import java.io.IOException; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.OptionalLong; -import java.util.UUID; -import java.util.stream.Collectors; -import jakarta.validation.Validator; -import jakarta.ws.rs.core.UriBuilder; - -import com.bakdata.conquery.apiv1.QueryProcessor; -import com.bakdata.conquery.apiv1.execution.ExecutionStatus; -import com.bakdata.conquery.apiv1.execution.OverviewExecutionStatus; -import com.bakdata.conquery.apiv1.execution.ResultAsset; -import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; -import com.bakdata.conquery.apiv1.query.CQElement; -import com.bakdata.conquery.apiv1.query.ConceptQuery; -import com.bakdata.conquery.apiv1.query.Query; -import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; -import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; -import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; -import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; -import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.mode.cluster.ClusterNamespaceHandler; -import com.bakdata.conquery.mode.cluster.ClusterState; -import com.bakdata.conquery.mode.cluster.InternalMapperFactory; -import com.bakdata.conquery.models.auth.AuthorizationController; -import com.bakdata.conquery.models.auth.entities.User; -import com.bakdata.conquery.models.auth.permissions.AbilitySets; -import com.bakdata.conquery.models.auth.permissions.ExecutionPermission; -import com.bakdata.conquery.models.config.ArrowResultProvider; -import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.config.CsvResultProvider; -import com.bakdata.conquery.models.config.ExcelResultProvider; -import com.bakdata.conquery.models.config.ParquetResultProvider; -import com.bakdata.conquery.models.datasets.Dataset; -import com.bakdata.conquery.models.datasets.SecondaryIdDescription; -import com.bakdata.conquery.models.execution.ExecutionState; -import com.bakdata.conquery.models.execution.ManagedExecution; -import com.bakdata.conquery.models.forms.managed.AbsoluteFormQuery; -import com.bakdata.conquery.models.forms.managed.ManagedForm; -import com.bakdata.conquery.models.forms.managed.ManagedInternalForm; -import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; -import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId; -import com.bakdata.conquery.models.index.IndexService; -import com.bakdata.conquery.models.query.DistributedExecutionManager; -import com.bakdata.conquery.models.query.ManagedQuery; -import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.models.worker.DatasetRegistry; -import com.bakdata.conquery.models.worker.DistributedNamespace; -import com.bakdata.conquery.util.extensions.MetaStorageExtension; -import com.bakdata.conquery.util.extensions.UserExtension; -import com.google.common.collect.ImmutableList; -import io.dropwizard.core.setup.Environment; -import io.dropwizard.jersey.validation.Validators; -import lombok.SneakyThrows; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -public class StoredQueriesProcessorTest { - - public static final ConqueryConfig CONFIG = new ConqueryConfig(); - public static final UriBuilder URI_BUILDER = UriBuilder.fromPath("http://localhost"); - public static final IndexService INDEX_SERVICE = new IndexService(CONFIG.getCsv().createCsvParserSettings()); - private static final Environment ENVIRONMENT = new Environment("StoredQueriesProcessorTest"); - @RegisterExtension - private static final MetaStorageExtension STORAGE_EXTENSION = new MetaStorageExtension(ENVIRONMENT.metrics()); - public static final MetaStorage STORAGE = STORAGE_EXTENSION.getMetaStorage(); - @RegisterExtension - private static final UserExtension USER_0_EXTENSIONS = new UserExtension(STORAGE, "0"); - @RegisterExtension - private static final UserExtension USER_1_EXTENSIONS = new UserExtension(STORAGE, "1"); - private static final User[] USERS = new User[]{ - USER_0_EXTENSIONS.getUser(), - USER_1_EXTENSIONS.getUser() - }; - - private static final Validator VALIDATOR = Validators.newValidator(); - public static final InternalMapperFactory INTERNAL_MAPPER_FACTORY = new InternalMapperFactory(CONFIG, VALIDATOR); - private static final DatasetRegistry - DATASET_REGISTRY = - new DatasetRegistry<>( - CONFIG, - INTERNAL_MAPPER_FACTORY, - new ClusterNamespaceHandler(new ClusterState(), CONFIG, INTERNAL_MAPPER_FACTORY), - INDEX_SERVICE - ); - private static final QueryProcessor processor = new QueryProcessor(DATASET_REGISTRY, STORAGE, CONFIG, VALIDATOR); - private static final ExcelResultProvider EXCEL_RESULT_PROVIDER = new ExcelResultProvider(); - private static final CsvResultProvider CSV_RESULT_PROVIDER = new CsvResultProvider(); - private static final ArrowResultProvider ARROW_RESULT_PROVIDER = new ArrowResultProvider(); - private static final ParquetResultProvider PARQUET_RESULT_PROVIDER = new ParquetResultProvider(); - private static final Dataset DATASET_0 = new Dataset() {{ - setName("dataset0"); - }}; - private static final Dataset DATASET_1 = new Dataset() {{ - setName("dataset1"); - }}; - private static ManagedExecutionId QUERY_ID_0; - private static ManagedExecutionId QUERY_ID_1; - private static ManagedExecutionId QUERY_ID_2; - private static ManagedExecutionId QUERY_ID_3; - private static ManagedExecutionId QUERY_ID_4; - private static ManagedExecutionId QUERY_ID_5; - private static ManagedExecutionId QUERY_ID_6; - private static ManagedExecutionId QUERY_ID_7; - private static ManagedExecutionId QUERY_ID_8; - private static ManagedExecutionId QUERY_ID_9; - private static List QUERIES; - - @BeforeAll - public static void beforeAll() throws IOException { - new AuthorizationController(STORAGE, CONFIG, new Environment(StoredQueriesProcessorTest.class.getSimpleName()), null); - - DATASET_REGISTRY.createNamespace(DATASET_0, STORAGE, ENVIRONMENT); - DATASET_REGISTRY.createNamespace(DATASET_1, STORAGE, ENVIRONMENT); - - - QUERY_ID_0 = createExecutionId(DATASET_0, "0"); - QUERY_ID_1 = createExecutionId(DATASET_1, "1"); - QUERY_ID_2 = createExecutionId(DATASET_0, "2"); - QUERY_ID_3 = createExecutionId(DATASET_0, "3"); - QUERY_ID_4 = createExecutionId(DATASET_0, "4"); - QUERY_ID_5 = createExecutionId(DATASET_0, "5"); - QUERY_ID_6 = createExecutionId(DATASET_0, "6"); - QUERY_ID_7 = createExecutionId(DATASET_0, "7"); - QUERY_ID_8 = createExecutionId(DATASET_0, "8"); - QUERY_ID_9 = createExecutionId(DATASET_0, "9"); - - User user0 = USER_0_EXTENSIONS.getUser(); - for (ManagedExecutionId id : List.of(QUERY_ID_4, QUERY_ID_7, QUERY_ID_8, QUERY_ID_9)) { - - user0.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, id)); - } - - - QUERIES = ImmutableList.of( - mockManagedConceptQueryFrontEnd(USERS[0], QUERY_ID_0, NEW, DATASET_0, 100L), // included - mockManagedConceptQueryFrontEnd(USERS[0], QUERY_ID_1, NEW, DATASET_1, 100L), // not included: wrong dataset - mockManagedForm(USERS[0], QUERY_ID_2, NEW, DATASET_0), // not included: not a ManagedQuery - mockManagedConceptQueryFrontEnd(USERS[1], QUERY_ID_3, NEW, DATASET_0, 100L), // not included: missing permission - mockManagedConceptQueryFrontEnd(USERS[1], QUERY_ID_4, DONE, DATASET_0, 100L), // included - mockManagedConceptQueryFrontEnd(USERS[0], QUERY_ID_5, FAILED, DATASET_0, 100L), // not included: wrong state - mockManagedQuery(new AbsoluteFormQuery(null, null, null, null), - USERS[0], - QUERY_ID_6, - NEW, - DATASET_0, - 100L - ), // not included: wrong query structure - mockManagedSecondaryIdQueryFrontEnd(USERS[1], QUERY_ID_7, DONE, new CQAnd() {{ - setChildren(List.of(new CQConcept() {{ - setLabel("User Concept Label"); - }})); - }}, DATASET_0 - ), // included, but secondaryId-Query - mockManagedQuery(new ConceptQuery(new CQExternal(new ArrayList<>(), new String[0][0], false)), - USERS[1], - QUERY_ID_8, - DONE, - DATASET_0, - 100L - ), // included - mockManagedConceptQueryFrontEnd(USERS[1], - QUERY_ID_9, - DONE, - DATASET_0, - 2_000_000L - ) // included, but no result url for xlsx (result has too many rows) - - ); - - QUERIES.forEach(STORAGE::addExecution); - } - - private static void setState(ExecutionState execState, ManagedExecutionId id) { - if (execState != NEW) { - DistributedExecutionManager.DistributedExecutionInfo state = new DistributedExecutionManager.DistributedExecutionInfo(Collections.emptyList()); - state.setExecutionState(execState); - state.getExecutingLock().countDown(); - - DATASET_REGISTRY.get(id.getDataset()).getExecutionManager().addState(id, state); - } - } - - private static ManagedExecutionId createExecutionId(Dataset dataset0, String s) { - StringBuilder idBuilder = new StringBuilder("00000000-0000-0000-0000-000000000000"); - idBuilder.replace(idBuilder.length() - s.length(), idBuilder.length(), s); - - return new ManagedExecutionId(dataset0.getId(), UUID.fromString(idBuilder.toString())); - } - - private static ManagedQuery mockManagedConceptQueryFrontEnd(User user, ManagedExecutionId id, ExecutionState execState, Dataset dataset, long resultCount) { - return mockManagedQuery( - new ConceptQuery( - new CQAnd() {{ - // Shorthand class initializer block to support visiting of CQAnd Children - // We set a user label here, so that no ConceptId needs to be resolved - setChildren(List.of(new CQConcept() {{ - setLabel("User Concept Label"); - }})); - }} - ), - user, - id, - execState, dataset, resultCount - ); - } - - private static ManagedForm mockManagedForm(User user, ManagedExecutionId id, ExecutionState execState, final Dataset dataset) { - return new ManagedInternalForm<>(new ExportForm(), user.getId(), dataset.getId(), STORAGE, DATASET_REGISTRY, CONFIG) { - { - setState(execState, id); - setCreationTime(LocalDateTime.MIN); - setQueryId(id.getExecution()); - } - }; - } - - private static ManagedQuery mockManagedQuery( - Query queryDescription, - User user, - ManagedExecutionId id, - ExecutionState execState, - final Dataset dataset, - final long resultCount) { - ManagedQuery managedQuery = new ManagedQuery(queryDescription, user.getId(), dataset.getId(), STORAGE, DATASET_REGISTRY, CONFIG) { - { - setCreationTime(LocalDateTime.MIN); - setQueryId(id.getExecution()); - setConfig(CONFIG); - } - - @Override - public List collectResultInfos() { - // With method is mocked because the ExcelResultProvider needs some info to check dimensions, - // but actually resolving the query here requires much more setup - return Collections.emptyList(); - } - - @Override - public synchronized OptionalLong resultRowCount() { - return OptionalLong.of(resultCount); - } - }; - setState(execState, managedQuery.getId()); - return managedQuery; - } - - private static ManagedQuery mockManagedSecondaryIdQueryFrontEnd(User user, ManagedExecutionId id, ExecutionState execState, CQElement root, Dataset dataset) { - final SecondaryIdQuery sIdQ = new SecondaryIdQuery(); - SecondaryIdDescription sId = new SecondaryIdDescription() {{ - setDataset(dataset.getId()); - setName("sid"); - }}; - sIdQ.setSecondaryId(sId.getId()); - sIdQ.setRoot(root); - - return mockManagedQuery(sIdQ, user, id, execState, dataset, 100L); - } - - @Test - public void getQueriesFiltered() { - - List infos = processor.getQueriesFiltered(DATASET_0.getId(), URI_BUILDER, USERS[0], QUERIES.stream().map(ManagedExecution::getId), true) - .collect(Collectors.toList()); - - assertThat(infos) - .containsExactly( - makeState(QUERY_ID_0, USERS[0], USERS[0], NEW, "CONCEPT_QUERY", null, null), - makeState(QUERY_ID_4, USERS[1], USERS[0], DONE, "CONCEPT_QUERY", null, 100L), - makeState(QUERY_ID_7, USERS[1], USERS[0], DONE, "SECONDARY_ID_QUERY", new SecondaryIdDescriptionId(DATASET_0.getId(), "sid"), 100L), - makeState(QUERY_ID_8, USERS[1], USERS[0], DONE, "CONCEPT_QUERY", null, 100L), - makeState(QUERY_ID_9, USERS[1], USERS[0], DONE, "CONCEPT_QUERY", null, 2_000_000L) - ); - } - - @SneakyThrows - private static ExecutionStatus makeState( - ManagedExecutionId id, - User owner, - User callingUser, - ExecutionState state, - String typeLabel, - SecondaryIdDescriptionId secondaryId, - Long resultCount) { - OverviewExecutionStatus status = new OverviewExecutionStatus(); - - final ManagedQuery execMock = new ManagedQuery(null, owner.getId(), DATASET_0.getId(), STORAGE, DATASET_REGISTRY, CONFIG) { - { - setQueryId(id.getExecution()); - setConfig(CONFIG); - } - - @Override - public List collectResultInfos() { - return Collections.emptyList(); - } - - @Override - public synchronized OptionalLong resultRowCount() { - return resultCount == null ? OptionalLong.empty(): OptionalLong.of(resultCount); - } - }; - - status.setTags(new String[0]); - status.setLabel(id.getExecution().toString()); - status.setPristineLabel(true); - status.setCreatedAt(LocalDateTime.MIN.atZone(ZoneId.systemDefault())); - status.setOwner(owner.getId()); - status.setOwnerName(owner.getLabel()); - status.setShared(false); - status.setOwn(owner.equals(callingUser)); - status.setId(id); - status.setStatus(state); - status.setQueryType(typeLabel); - status.setNumberOfResults(resultCount); - status.setSecondaryId(secondaryId); // This is probably not interesting on the overview (only if there is a filter for the search) - if (state.equals(DONE)) { - List resultUrls = new ArrayList<>(); - resultUrls.addAll(EXCEL_RESULT_PROVIDER.generateResultURLs(execMock, URI_BUILDER.clone(), true)); - resultUrls.addAll(CSV_RESULT_PROVIDER.generateResultURLs(execMock, URI_BUILDER.clone(), true)); - resultUrls.addAll(ARROW_RESULT_PROVIDER.generateResultURLs(execMock, URI_BUILDER.clone(), true)); - resultUrls.addAll(PARQUET_RESULT_PROVIDER.generateResultURLs(execMock, URI_BUILDER.clone(), true)); - status.setResultUrls(resultUrls); - } - - return status; - } - -} +package com.bakdata.conquery.api; + +import static com.bakdata.conquery.models.execution.ExecutionState.*; +import static org.assertj.core.api.Assertions.assertThat; + +import jakarta.validation.Validator; +import jakarta.ws.rs.core.UriBuilder; +import java.io.IOException; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.OptionalLong; +import java.util.UUID; +import java.util.stream.Collectors; + +import com.bakdata.conquery.apiv1.QueryProcessor; +import com.bakdata.conquery.apiv1.execution.ExecutionStatus; +import com.bakdata.conquery.apiv1.execution.OverviewExecutionStatus; +import com.bakdata.conquery.apiv1.execution.ResultAsset; +import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; +import com.bakdata.conquery.apiv1.query.CQElement; +import com.bakdata.conquery.apiv1.query.ConceptQuery; +import com.bakdata.conquery.apiv1.query.Query; +import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; +import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; +import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; +import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.mode.cluster.ClusterNamespaceHandler; +import com.bakdata.conquery.mode.cluster.ClusterState; +import com.bakdata.conquery.mode.cluster.InternalMapperFactory; +import com.bakdata.conquery.models.auth.AuthorizationController; +import com.bakdata.conquery.models.auth.entities.User; +import com.bakdata.conquery.models.auth.permissions.AbilitySets; +import com.bakdata.conquery.models.auth.permissions.ExecutionPermission; +import com.bakdata.conquery.models.config.ArrowResultProvider; +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.config.CsvResultProvider; +import com.bakdata.conquery.models.config.ExcelResultProvider; +import com.bakdata.conquery.models.config.ParquetResultProvider; +import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.datasets.SecondaryIdDescription; +import com.bakdata.conquery.models.execution.ExecutionState; +import com.bakdata.conquery.models.execution.ManagedExecution; +import com.bakdata.conquery.models.forms.managed.AbsoluteFormQuery; +import com.bakdata.conquery.models.forms.managed.ManagedForm; +import com.bakdata.conquery.models.forms.managed.ManagedInternalForm; +import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; +import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId; +import com.bakdata.conquery.models.index.IndexService; +import com.bakdata.conquery.models.query.DistributedExecutionManager; +import com.bakdata.conquery.models.query.ManagedQuery; +import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.DistributedNamespace; +import com.bakdata.conquery.util.extensions.MetaStorageExtension; +import com.bakdata.conquery.util.extensions.UserExtension; +import com.google.common.collect.ImmutableList; +import io.dropwizard.core.setup.Environment; +import io.dropwizard.jersey.validation.Validators; +import lombok.SneakyThrows; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +public class StoredQueriesProcessorTest { + + public static final ConqueryConfig CONFIG = new ConqueryConfig(); + public static final UriBuilder URI_BUILDER = UriBuilder.fromPath("http://localhost"); + public static final IndexService INDEX_SERVICE = new IndexService(CONFIG.getCsv().createCsvParserSettings()); + private static final Environment ENVIRONMENT = new Environment("StoredQueriesProcessorTest"); + @RegisterExtension + private static final MetaStorageExtension STORAGE_EXTENSION = new MetaStorageExtension(ENVIRONMENT.metrics()); + public static final MetaStorage STORAGE = STORAGE_EXTENSION.getMetaStorage(); + @RegisterExtension + private static final UserExtension USER_0_EXTENSIONS = new UserExtension(STORAGE, "0"); + @RegisterExtension + private static final UserExtension USER_1_EXTENSIONS = new UserExtension(STORAGE, "1"); + private static final User[] USERS = new User[]{USER_0_EXTENSIONS.getUser(), USER_1_EXTENSIONS.getUser() + }; + + private static final Validator VALIDATOR = Validators.newValidator(); + public static final InternalMapperFactory INTERNAL_MAPPER_FACTORY = new InternalMapperFactory(CONFIG, VALIDATOR); + private static final DatasetRegistry DATASET_REGISTRY = new DatasetRegistry<>( + CONFIG, + INTERNAL_MAPPER_FACTORY, + new ClusterNamespaceHandler(new ClusterState(), CONFIG, INTERNAL_MAPPER_FACTORY), + INDEX_SERVICE + ); + private static final QueryProcessor processor = new QueryProcessor(DATASET_REGISTRY, STORAGE, CONFIG, VALIDATOR); + private static final ExcelResultProvider EXCEL_RESULT_PROVIDER = new ExcelResultProvider(); + private static final CsvResultProvider CSV_RESULT_PROVIDER = new CsvResultProvider(); + private static final ArrowResultProvider ARROW_RESULT_PROVIDER = new ArrowResultProvider(); + private static final ParquetResultProvider PARQUET_RESULT_PROVIDER = new ParquetResultProvider(); + private static final Dataset DATASET_0 = new Dataset() { + { + setName("dataset0"); + } + }; + private static final Dataset DATASET_1 = new Dataset() { + { + setName("dataset1"); + } + }; + private static ManagedExecutionId QUERY_ID_0; + private static ManagedExecutionId QUERY_ID_1; + private static ManagedExecutionId QUERY_ID_2; + private static ManagedExecutionId QUERY_ID_3; + private static ManagedExecutionId QUERY_ID_4; + private static ManagedExecutionId QUERY_ID_5; + private static ManagedExecutionId QUERY_ID_6; + private static ManagedExecutionId QUERY_ID_7; + private static ManagedExecutionId QUERY_ID_8; + private static ManagedExecutionId QUERY_ID_9; + private static List QUERIES; + + @BeforeAll + public static void beforeAll() throws IOException { + new AuthorizationController( + STORAGE, + CONFIG, + new Environment(StoredQueriesProcessorTest.class.getSimpleName()), + null); + + DATASET_REGISTRY.createNamespace(DATASET_0, STORAGE, ENVIRONMENT); + DATASET_REGISTRY.createNamespace(DATASET_1, STORAGE, ENVIRONMENT); + + + QUERY_ID_0 = createExecutionId(DATASET_0, "0"); + QUERY_ID_1 = createExecutionId(DATASET_1, "1"); + QUERY_ID_2 = createExecutionId(DATASET_0, "2"); + QUERY_ID_3 = createExecutionId(DATASET_0, "3"); + QUERY_ID_4 = createExecutionId(DATASET_0, "4"); + QUERY_ID_5 = createExecutionId(DATASET_0, "5"); + QUERY_ID_6 = createExecutionId(DATASET_0, "6"); + QUERY_ID_7 = createExecutionId(DATASET_0, "7"); + QUERY_ID_8 = createExecutionId(DATASET_0, "8"); + QUERY_ID_9 = createExecutionId(DATASET_0, "9"); + + User user0 = USER_0_EXTENSIONS.getUser(); + for (ManagedExecutionId id : List.of(QUERY_ID_4, QUERY_ID_7, QUERY_ID_8, QUERY_ID_9)) { + + user0.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, id)); + } + + + QUERIES = ImmutableList.of( + mockManagedConceptQueryFrontEnd(USERS[0], QUERY_ID_0, NEW, DATASET_0, 100L), // included + mockManagedConceptQueryFrontEnd(USERS[0], QUERY_ID_1, NEW, DATASET_1, 100L), // not included: wrong dataset + mockManagedForm(USERS[0], QUERY_ID_2, NEW, DATASET_0), // not included: not a ManagedQuery + mockManagedConceptQueryFrontEnd(USERS[1], QUERY_ID_3, NEW, DATASET_0, 100L), // not included: missing permission + mockManagedConceptQueryFrontEnd(USERS[1], QUERY_ID_4, DONE, DATASET_0, 100L), // included + mockManagedConceptQueryFrontEnd(USERS[0], QUERY_ID_5, FAILED, DATASET_0, 100L), // not included: wrong state + mockManagedQuery(new AbsoluteFormQuery(null, null, null, null), USERS[0], QUERY_ID_6, NEW, DATASET_0, 100L + ), // not included: wrong query structure + mockManagedSecondaryIdQueryFrontEnd(USERS[1], QUERY_ID_7, DONE, new CQAnd() { + { + setChildren(List.of(new CQConcept() { + { + setLabel("User Concept Label"); + } + })); + } + }, DATASET_0 + ), // included, but secondaryId-Query + mockManagedQuery( + new ConceptQuery(new CQExternal(new ArrayList<>(), new String[0][0], false)), + USERS[1], + QUERY_ID_8, + DONE, + DATASET_0, + 100L + ), // included + mockManagedConceptQueryFrontEnd(USERS[1], QUERY_ID_9, DONE, DATASET_0, 2_000_000L + ) // included, but no result url for xlsx (result has too many rows) + + ); + + QUERIES.forEach(STORAGE::addExecution); + } + + private static void setState(ExecutionState execState, ManagedExecutionId id) { + if (execState != NEW) { + DistributedExecutionManager.DistributedExecutionInfo state = new DistributedExecutionManager.DistributedExecutionInfo( + Collections.emptyList()); + state.setExecutionState(execState); + state.getExecutingLock().countDown(); + + DATASET_REGISTRY.get(id.getDataset()).getExecutionManager().addState(id, state); + } + } + + private static ManagedExecutionId createExecutionId(Dataset dataset0, String s) { + StringBuilder idBuilder = new StringBuilder("00000000-0000-0000-0000-000000000000"); + idBuilder.replace(idBuilder.length() - s.length(), idBuilder.length(), s); + + return new ManagedExecutionId(dataset0.getId(), UUID.fromString(idBuilder.toString())); + } + + private static ManagedQuery mockManagedConceptQueryFrontEnd( + User user, + ManagedExecutionId id, + ExecutionState execState, + Dataset dataset, + long resultCount) { + return mockManagedQuery( + new ConceptQuery( + new CQAnd() { + { + // Shorthand class initializer block to support visiting of CQAnd Children + // We set a user label here, so that no ConceptId needs to be resolved + setChildren(List.of(new CQConcept() { + { + setLabel("User Concept Label"); + } + })); + } + } + ), + user, + id, + execState, + dataset, + resultCount + ); + } + + private static ManagedForm mockManagedForm( + User user, + ManagedExecutionId id, + ExecutionState execState, + final Dataset dataset) { + return new ManagedInternalForm<>( + new ExportForm(), + user.getId(), + dataset.getId(), + STORAGE, + DATASET_REGISTRY, + CONFIG) { + { + setState(execState, id); + setCreationTime(LocalDateTime.MIN); + setQueryId(id.getExecution()); + } + }; + } + + private static ManagedQuery mockManagedQuery( + Query queryDescription, + User user, + ManagedExecutionId id, + ExecutionState execState, + final Dataset dataset, + final long resultCount) { + ManagedQuery managedQuery = new ManagedQuery( + queryDescription, + user.getId(), + dataset.getId(), + STORAGE, + DATASET_REGISTRY, + CONFIG) { + { + setCreationTime(LocalDateTime.MIN); + setQueryId(id.getExecution()); + setConfig(CONFIG); + } + + @Override + public List collectResultInfos() { + // With method is mocked because the ExcelResultProvider needs some info to check dimensions, + // but actually resolving the query here requires much more setup + return Collections.emptyList(); + } + + @Override + public synchronized OptionalLong resultRowCount() { + return OptionalLong.of(resultCount); + } + }; + setState(execState, managedQuery.getId()); + return managedQuery; + } + + private static ManagedQuery mockManagedSecondaryIdQueryFrontEnd( + User user, + ManagedExecutionId id, + ExecutionState execState, + CQElement root, + Dataset dataset) { + final SecondaryIdQuery sIdQ = new SecondaryIdQuery(); + SecondaryIdDescription sId = new SecondaryIdDescription() { + { + setDataset(dataset.getId()); + setName("sid"); + } + }; + sIdQ.setSecondaryId(sId.getId()); + sIdQ.setRoot(root); + + return mockManagedQuery(sIdQ, user, id, execState, dataset, 100L); + } + + @Test + public void getQueriesFiltered() { + + List infos = processor.getQueriesFiltered( + DATASET_0.getId(), + URI_BUILDER, + USERS[0], + QUERIES.stream().map(ManagedExecution::getId), + true).collect(Collectors.toList()); + + assertThat(infos).containsExactly( + makeState(QUERY_ID_0, USERS[0], USERS[0], NEW, "CONCEPT_QUERY", null, null), + makeState(QUERY_ID_4, USERS[1], USERS[0], DONE, "CONCEPT_QUERY", null, 100L), + makeState( + QUERY_ID_7, + USERS[1], + USERS[0], + DONE, + "SECONDARY_ID_QUERY", + new SecondaryIdDescriptionId(DATASET_0.getId(), "sid"), + 100L), + makeState(QUERY_ID_8, USERS[1], USERS[0], DONE, "CONCEPT_QUERY", null, 100L), + makeState(QUERY_ID_9, USERS[1], USERS[0], DONE, "CONCEPT_QUERY", null, 2_000_000L) + ); + } + + @SneakyThrows + private static ExecutionStatus makeState( + ManagedExecutionId id, + User owner, + User callingUser, + ExecutionState state, + String typeLabel, + SecondaryIdDescriptionId secondaryId, + Long resultCount) { + OverviewExecutionStatus status = new OverviewExecutionStatus(); + + final ManagedQuery execMock = new ManagedQuery( + null, + owner.getId(), + DATASET_0.getId(), + STORAGE, + DATASET_REGISTRY, + CONFIG) { + { + setQueryId(id.getExecution()); + setConfig(CONFIG); + } + + @Override + public List collectResultInfos() { + return Collections.emptyList(); + } + + @Override + public synchronized OptionalLong resultRowCount() { + return resultCount == null ? OptionalLong.empty() : OptionalLong.of(resultCount); + } + }; + + status.setTags(new String[0]); + status.setLabel(id.getExecution().toString()); + status.setPristineLabel(true); + status.setCreatedAt(LocalDateTime.MIN.atZone(ZoneId.systemDefault())); + status.setOwner(owner.getId()); + status.setOwnerName(owner.getLabel()); + status.setShared(false); + status.setOwn(owner.equals(callingUser)); + status.setId(id); + status.setStatus(state); + status.setQueryType(typeLabel); + status.setNumberOfResults(resultCount); + status.setSecondaryId(secondaryId); // This is probably not interesting on the overview (only if there is a filter for the search) + if (state.equals(DONE)) { + List resultUrls = new ArrayList<>(); + resultUrls.addAll(EXCEL_RESULT_PROVIDER.generateResultURLs(execMock, URI_BUILDER.clone(), true)); + resultUrls.addAll(CSV_RESULT_PROVIDER.generateResultURLs(execMock, URI_BUILDER.clone(), true)); + resultUrls.addAll(ARROW_RESULT_PROVIDER.generateResultURLs(execMock, URI_BUILDER.clone(), true)); + resultUrls.addAll(PARQUET_RESULT_PROVIDER.generateResultURLs(execMock, URI_BUILDER.clone(), true)); + status.setResultUrls(resultUrls); + } + + return status; + } + +} diff --git a/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java b/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java index 57f58e634ac..55c9acd090d 100644 --- a/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java +++ b/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java @@ -5,6 +5,7 @@ import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.when; +import jakarta.validation.Validator; import java.net.URI; import java.time.ZoneId; import java.util.Collections; @@ -12,7 +13,6 @@ import java.util.Map; import java.util.UUID; import java.util.stream.Stream; -import jakarta.validation.Validator; import com.bakdata.conquery.apiv1.FormConfigPatch; import com.bakdata.conquery.apiv1.forms.FormConfigAPI; @@ -106,11 +106,9 @@ public void setupTestClass() throws Exception { if (id.equals(datasetId)) { when(namespaceMock.getDataset()).thenReturn(dataset); - } - else if (id.equals(datasetId1)) { + } else if (id.equals(datasetId1)) { when(namespaceMock.getDataset()).thenReturn(dataset1); - } - else { + } else { throw new IllegalStateException("Unknown dataset id."); } return namespaceMock; @@ -120,9 +118,15 @@ else if (id.equals(datasetId1)) { when(namespacesMock.inject(any(MutableInjectableValues.class))).thenCallRealMethod(); - ((MutableInjectableValues) FormConfigProcessor.getMAPPER().getInjectableValues()).add(NamespacedStorageProvider.class, namespacesMock); + ((MutableInjectableValues) FormConfigProcessor.getMAPPER().getInjectableValues()).add( + NamespacedStorageProvider.class, + namespacesMock); processor = new FormConfigProcessor(validator, metaStorage, namespacesMock); - AuthorizationController controller = new AuthorizationController(metaStorage, config, new Environment(this.getClass().getSimpleName()), null); + AuthorizationController controller = new AuthorizationController( + metaStorage, + config, + new Environment(this.getClass().getSimpleName()), + null); controller.start(); } @@ -130,7 +134,13 @@ else if (id.equals(datasetId1)) { @BeforeEach public void setupTest() { - final ManagedQuery managedQuery = new ManagedQuery(null, new UserId("test"), dataset.getId(), metaStorage, null, config); + final ManagedQuery managedQuery = new ManagedQuery( + null, + new UserId("test"), + dataset.getId(), + metaStorage, + null, + config); managedQuery.setMetaStorage(metaStorage); managedQuery.setQueryId(UUID.randomUUID()); @@ -158,7 +168,11 @@ public void addConfigWithoutTranslation() { user.addPermission(dataset.createPermission(Ability.READ.asSet())); ObjectMapper mapper = FormConfigProcessor.getMAPPER(); - FormConfigAPI formConfig = FormConfigAPI.builder().formType(form.getFormType()).values(mapper.valueToTree(form)).build(); + FormConfigAPI formConfig = FormConfigAPI.builder() + .formType(form.getFormType()) + .values( + mapper.valueToTree(form)) + .build(); processor.addConfig(user, dataset.getId(), formConfig); @@ -171,7 +185,9 @@ public void deleteConfig() { user.addPermission(DatasetPermission.onInstance(Ability.READ, datasetId)); ObjectMapper mapper = FormConfigProcessor.getMAPPER(); - FormConfig formConfig = new FormConfig(form.getClass().getAnnotation(CPSType.class).id(), mapper.valueToTree(form)); + FormConfig formConfig = new FormConfig( + form.getClass().getAnnotation(CPSType.class).id(), + mapper.valueToTree(form)); formConfig.setDataset(dataset.getId()); formConfig.setOwner(user.getId()); formConfig.setMetaStorage(metaStorage); @@ -185,7 +201,8 @@ public void deleteConfig() { // CHECK assertThat(metaStorage.getAllFormConfigs()).doesNotContain(formConfig); - assertThat(user.getPermissions()).doesNotContain(FormConfigPermission.onInstance(AbilitySets.FORM_CONFIG_CREATOR, formConfig.getId())); + assertThat(user.getPermissions()).doesNotContain( + FormConfigPermission.onInstance(AbilitySets.FORM_CONFIG_CREATOR, formConfig.getId())); } @Test @@ -208,19 +225,24 @@ public void getConfig() { // CHECK assertThat(response).usingRecursiveComparison() - .ignoringFields(FormConfigOverviewRepresentation.Fields.createdAt) - .isEqualTo(FormConfigFullRepresentation.builder() - .formType(form.getClass().getAnnotation(CPSType.class).id()) - .id(formConfig.getId()) - .label(formConfig.getLabel()) - .own(true) - .ownerName(user.getLabel()) - .shared(false) - .groups(Collections.emptySet()) - .system(false) - .tags(formConfig.getTags()) - .values(values) - .build()); + .ignoringFields( + FormConfigOverviewRepresentation.Fields.createdAt) + .isEqualTo( + FormConfigFullRepresentation.builder() + .formType(form.getClass().getAnnotation(CPSType.class).id()) + .id( + formConfig.getId()) + .label(formConfig.getLabel()) + .own(true) + .ownerName(user.getLabel()) + .shared( + false) + .groups(Collections.emptySet()) + .system(false) + .tags(formConfig.getTags()) + .values( + values) + .build()); } @@ -230,14 +252,21 @@ public void formScannerTest() throws Exception { formScanner.execute(Collections.emptyMap(), null); - assertThat(FormScanner.FRONTEND_FORM_CONFIGS).containsKeys("TEST_FORM_REL_URL", "TEST_FORM_ABS_URL", "EXPORT_FORM", "FULL_EXPORT_FORM"); + assertThat(FormScanner.FRONTEND_FORM_CONFIGS).containsKeys( + "TEST_FORM_REL_URL", + "TEST_FORM_ABS_URL", + "EXPORT_FORM", + "FULL_EXPORT_FORM"); - assertThat(FormScanner.FRONTEND_FORM_CONFIGS.get("TEST_FORM_REL_URL").getRawConfig().get("manualUrl").asText()).as("relative to base url resolved url") - .isEqualTo("http://example.org/manual/test-form"); + assertThat( + FormScanner.FRONTEND_FORM_CONFIGS.get("TEST_FORM_REL_URL").getRawConfig().get("manualUrl").asText()).as( + "relative to base url resolved url").isEqualTo("http://example.org/manual/test-form"); - assertThat(FormScanner.FRONTEND_FORM_CONFIGS.get("TEST_FORM_ABS_URL").getRawConfig().get("manualUrl").asText()).as("given absolute url") - .isEqualTo( - "http://example.org/absolute-url/test-form"); + assertThat( + FormScanner.FRONTEND_FORM_CONFIGS.get("TEST_FORM_ABS_URL").getRawConfig().get("manualUrl").asText()).as( + "given absolute url") + .isEqualTo( + "http://example.org/absolute-url/test-form"); } @@ -257,43 +286,67 @@ public void getConfigs() { ObjectMapper mapper = FormConfigProcessor.getMAPPER(); - FormConfigAPI formConfig = FormConfigAPI.builder().formType(form.getFormType()).values(mapper.valueToTree(form)).build(); - FormConfigAPI formConfig2 = FormConfigAPI.builder().formType(form2.getFormType()).values(mapper.valueToTree(form2)).build(); + FormConfigAPI formConfig = FormConfigAPI.builder() + .formType(form.getFormType()) + .values( + mapper.valueToTree(form)) + .build(); + FormConfigAPI formConfig2 = FormConfigAPI.builder() + .formType(form2.getFormType()) + .values( + mapper.valueToTree(form2)) + .build(); // This should not be retrieved by the user because it does not hold the Permission to create TestForms - FormConfigAPI formConfig3 = FormConfigAPI.builder().formType(form3.getFormType()).values(mapper.valueToTree(form2)).build(); + FormConfigAPI formConfig3 = FormConfigAPI.builder() + .formType(form3.getFormType()) + .values( + mapper.valueToTree(form2)) + .build(); FormConfigId formId = processor.addConfig(user, dataset.getId(), formConfig).getId(); FormConfigId formId2 = processor.addConfig(user, dataset.getId(), formConfig2).getId(); FormConfigId formId3 = processor.addConfig(user, dataset.getId(), formConfig3).getId(); - FormScanner.FRONTEND_FORM_CONFIGS = Map.of(form.getFormType(), new FormType(form.getFormType(), new TextNode("dummy"))); + FormScanner.FRONTEND_FORM_CONFIGS = Map.of( + form.getFormType(), + new FormType(form.getFormType(), new TextNode("dummy"))); // EXECUTE - Stream response = processor.getConfigsByFormType(user, dataset.getId(), Collections.emptySet()); + Stream response = processor.getConfigsByFormType( + user, + dataset.getId(), + Collections.emptySet()); // CHECK - assertThat(response).containsExactlyInAnyOrder(FormConfigOverviewRepresentation.builder() - .formType(form.getClass().getAnnotation(CPSType.class).id()) - .id(formId) - .label(formConfig.getLabel()) - .own(true) - .ownerName(user.getLabel()) - .shared(false) - .system(false) - .tags(formConfig.getTags()) - .createdAt(formConfig.getCreationTime().atZone(ZoneId.systemDefault())) - .build(), - FormConfigOverviewRepresentation.builder() - .formType(form2.getClass().getAnnotation(CPSType.class).id()) - .id(formId2) - .label(formConfig2.getLabel()) - .own(true) - .ownerName(user.getLabel()) - .shared(false) - .system(false) - .tags(formConfig2.getTags()) - .createdAt(formConfig2.getCreationTime().atZone(ZoneId.systemDefault())) - .build() + assertThat(response).containsExactlyInAnyOrder( + FormConfigOverviewRepresentation.builder() + .formType(form.getClass().getAnnotation(CPSType.class).id()) + .id( + formId) + .label(formConfig.getLabel()) + .own(true) + .ownerName(user.getLabel()) + .shared(false) + .system( + false) + .tags(formConfig.getTags()) + .createdAt( + formConfig.getCreationTime().atZone(ZoneId.systemDefault())) + .build(), + FormConfigOverviewRepresentation.builder() + .formType(form2.getClass().getAnnotation(CPSType.class).id()) + .id( + formId2) + .label(formConfig2.getLabel()) + .own(true) + .ownerName(user.getLabel()) + .shared(false) + .system( + false) + .tags(formConfig2.getTags()) + .createdAt( + formConfig2.getCreationTime().atZone(ZoneId.systemDefault())) + .build() ); } @@ -317,14 +370,17 @@ public void patchConfig() { // EXECUTE PART 1 - processor.patchConfig(user, - config.getId(), - FormConfigPatch.builder() - .label("newTestLabel") - .tags(new String[]{"tag1", "tag2"}) - .groups(List.of(group1.getId())) - .values(new ObjectNode(mapper.getNodeFactory(), Map.of("test-Node", new TextNode("test-text")))) - .build() + processor.patchConfig( + user, + config.getId(), + FormConfigPatch.builder() + .label("newTestLabel") + .tags(new String[]{"tag1", "tag2"}) + .groups( + List.of(group1.getId())) + .values( + new ObjectNode(mapper.getNodeFactory(), Map.of("test-Node", new TextNode("test-text")))) + .build() ); // CHECK PART 1 @@ -335,14 +391,21 @@ public void patchConfig() { patchedFormExpected.setShared(true); patchedFormExpected.setTags(new String[]{"tag1", "tag2"}); patchedFormExpected.setOwner(user.getId()); - patchedFormExpected.setValues(new ObjectNode(mapper.getNodeFactory(), Map.of("test-Node", new TextNode("test-text")))); + patchedFormExpected.setValues( + new ObjectNode(mapper.getNodeFactory(), Map.of("test-Node", new TextNode("test-text")))); final String[] fieldsToIgnore = new String[]{FormConfig.Fields.creationTime, "cachedId", "metaStorage", "nsIdResolver"}; final FormConfigId formId = config.getId(); - assertThat(metaStorage.getFormConfig(formId)).usingRecursiveComparison().usingOverriddenEquals().ignoringFields(fieldsToIgnore).isEqualTo(patchedFormExpected); + assertThat(metaStorage.getFormConfig(formId)).usingRecursiveComparison() + .usingOverriddenEquals() + .ignoringFields( + fieldsToIgnore) + .isEqualTo(patchedFormExpected); - assertThat(metaStorage.getGroup(group1.getId()).getPermissions()).contains(FormConfigPermission.onInstance(AbilitySets.SHAREHOLDER, formId)); - assertThat(metaStorage.getGroup(group2.getId()).getPermissions()).doesNotContain(FormConfigPermission.onInstance(AbilitySets.SHAREHOLDER, formId)); + assertThat(metaStorage.getGroup(group1.getId()).getPermissions()).contains( + FormConfigPermission.onInstance(AbilitySets.SHAREHOLDER, formId)); + assertThat(metaStorage.getGroup(group2.getId()).getPermissions()).doesNotContain( + FormConfigPermission.onInstance(AbilitySets.SHAREHOLDER, formId)); // EXECUTE PART 2 (Unshare) @@ -351,10 +414,15 @@ public void patchConfig() { // CHECK PART 2 patchedFormExpected.setShared(false); - assertThat(metaStorage.getFormConfig(formId)).usingRecursiveComparison().ignoringFields(fieldsToIgnore).isEqualTo(patchedFormExpected); + assertThat(metaStorage.getFormConfig(formId)).usingRecursiveComparison() + .ignoringFields( + fieldsToIgnore) + .isEqualTo(patchedFormExpected); - assertThat(metaStorage.getGroup(group1.getId()).getPermissions()).doesNotContain(FormConfigPermission.onInstance(AbilitySets.SHAREHOLDER, formId)); - assertThat(metaStorage.getGroup(group2.getId()).getPermissions()).doesNotContain(FormConfigPermission.onInstance(AbilitySets.SHAREHOLDER, formId)); + assertThat(metaStorage.getGroup(group1.getId()).getPermissions()).doesNotContain( + FormConfigPermission.onInstance(AbilitySets.SHAREHOLDER, formId)); + assertThat(metaStorage.getGroup(group2.getId()).getPermissions()).doesNotContain( + FormConfigPermission.onInstance(AbilitySets.SHAREHOLDER, formId)); } } diff --git a/backend/src/test/java/com/bakdata/conquery/api/form/config/TestForm.java b/backend/src/test/java/com/bakdata/conquery/api/form/config/TestForm.java index bc9a92704aa..4c2c3cc33c9 100644 --- a/backend/src/test/java/com/bakdata/conquery/api/form/config/TestForm.java +++ b/backend/src/test/java/com/bakdata/conquery/api/form/config/TestForm.java @@ -26,7 +26,12 @@ public abstract class TestForm extends Form implements InternalForm { @Override - public ManagedExecution toManagedExecution(UserId user, DatasetId submittedDataset, MetaStorage storage, DatasetRegistry datasetRegistry, ConqueryConfig config) { + public ManagedExecution toManagedExecution( + UserId user, + DatasetId submittedDataset, + MetaStorage storage, + DatasetRegistry datasetRegistry, + ConqueryConfig config) { return new ManagedInternalForm<>(this, user, submittedDataset, storage, datasetRegistry, config); } diff --git a/backend/src/test/java/com/bakdata/conquery/apiv1/frontend/FrontendValueTest.java b/backend/src/test/java/com/bakdata/conquery/apiv1/frontend/FrontendValueTest.java index 3c7a4d13c95..662e2926611 100644 --- a/backend/src/test/java/com/bakdata/conquery/apiv1/frontend/FrontendValueTest.java +++ b/backend/src/test/java/com/bakdata/conquery/apiv1/frontend/FrontendValueTest.java @@ -31,4 +31,4 @@ public void testDistinctByNameOnly() { assertThat(Stream.of(firstA, secondA, similarA).distinct()).containsExactly(firstA, similarA); } -} \ No newline at end of file +} diff --git a/backend/src/test/java/com/bakdata/conquery/command/DistributedCommandsTest.java b/backend/src/test/java/com/bakdata/conquery/command/DistributedCommandsTest.java index 510561c2aa8..0348f7d16ca 100644 --- a/backend/src/test/java/com/bakdata/conquery/command/DistributedCommandsTest.java +++ b/backend/src/test/java/com/bakdata/conquery/command/DistributedCommandsTest.java @@ -3,11 +3,11 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; -import java.util.Map; -import java.util.concurrent.TimeUnit; import jakarta.ws.rs.client.Client; import jakarta.ws.rs.core.GenericType; import jakarta.ws.rs.core.Response; +import java.util.Map; +import java.util.concurrent.TimeUnit; import com.bakdata.conquery.Conquery; import com.bakdata.conquery.commands.ShardCommand; @@ -33,26 +33,30 @@ public class DistributedCommandsTest { public static final Duration CONNECT_RETRY_TIMEOUT = Duration.seconds(1); - private static final ConqueryConfig CONQUERY_CONFIG_MANAGER = new ConqueryConfig() {{ - ConfigOverride.configureRandomPorts(this); - this.setStorage(new NonPersistentStoreFactory()); - }}; - - private static final ConqueryConfig CONQUERY_CONFIG_SHARD = new ConqueryConfig() {{ - ConfigOverride.configureRandomPorts(this); - this.getCluster().setPort(CONQUERY_CONFIG_MANAGER.getCluster().getPort()); - this.getCluster().setConnectRetryTimeout(CONNECT_RETRY_TIMEOUT); - this.setStorage(new NonPersistentStoreFactory()); - }}; + private static final ConqueryConfig CONQUERY_CONFIG_MANAGER = new ConqueryConfig() { + { + ConfigOverride.configureRandomPorts(this); + this.setStorage(new NonPersistentStoreFactory()); + } + }; + + private static final ConqueryConfig CONQUERY_CONFIG_SHARD = new ConqueryConfig() { + { + ConfigOverride.configureRandomPorts(this); + this.getCluster().setPort(CONQUERY_CONFIG_MANAGER.getCluster().getPort()); + this.getCluster().setConnectRetryTimeout(CONNECT_RETRY_TIMEOUT); + this.setStorage(new NonPersistentStoreFactory()); + } + }; private static final DropwizardAppExtension SHARD = new DropwizardAppExtension<>( - Conquery.class, - CONQUERY_CONFIG_SHARD, - application -> new ShardCommand() + Conquery.class, + CONQUERY_CONFIG_SHARD, + application -> new ShardCommand() ); private static final DropwizardAppExtension MANAGER = new DropwizardAppExtension<>( - Conquery.class, - CONQUERY_CONFIG_MANAGER, - ServerCommand::new + Conquery.class, + CONQUERY_CONFIG_MANAGER, + ServerCommand::new ); @Test @@ -61,9 +65,7 @@ void checkHttpUpShard() { Client client = SHARD.client(); Response response = client.target( - String.format("http://localhost:%d/ping", SHARD.getAdminPort())) - .request() - .get(); + String.format("http://localhost:%d/ping", SHARD.getAdminPort())).request().get(); assertThat(response.getStatus()).isEqualTo(200); } @@ -74,9 +76,7 @@ void checkHttpUpManager() { Client client = MANAGER.client(); Response response = client.target( - String.format("http://localhost:%d/ping", MANAGER.getAdminPort())) - .request() - .get(); + String.format("http://localhost:%d/ping", MANAGER.getAdminPort())).request().get(); assertThat(response.getStatus()).isEqualTo(200); } @@ -89,10 +89,7 @@ void clusterEstablished() { // Wait for Shard to be connected await().atMost(5, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).untilAsserted(() -> { Response response = client.target( - String.format("http://localhost:%d/healthcheck", MANAGER.getAdminPort())) - .request() - .get(); - + String.format("http://localhost:%d/healthcheck", MANAGER.getAdminPort())).request().get(); assertThat(response.getStatus()).isEqualTo(200); @@ -102,7 +99,8 @@ void clusterEstablished() { assertThat(healthCheck).containsKey("cluster"); assertThat(healthCheck.get("cluster").healthy).isTrue(); - assertThat(healthCheck.get("cluster").getMessage()).isEqualTo(String.format(ClusterHealthCheck.HEALTHY_MESSAGE_FMT, 1)); + assertThat(healthCheck.get("cluster").getMessage()).isEqualTo( + String.format(ClusterHealthCheck.HEALTHY_MESSAGE_FMT, 1)); }); } diff --git a/backend/src/test/java/com/bakdata/conquery/commands/ManagerNodeTest.java b/backend/src/test/java/com/bakdata/conquery/commands/ManagerNodeTest.java index c2a30e92499..116445f6a5a 100644 --- a/backend/src/test/java/com/bakdata/conquery/commands/ManagerNodeTest.java +++ b/backend/src/test/java/com/bakdata/conquery/commands/ManagerNodeTest.java @@ -14,7 +14,6 @@ import ch.qos.logback.classic.Logger; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.read.ListAppender; - import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.models.worker.DatasetRegistry; @@ -34,8 +33,12 @@ void loadNamespacesLoadsEveryDiscoveredNamespace() { NamespaceStorage firstStorage = createStorage("dataset_first"); NamespaceStorage secondStorage = createStorage("dataset_second"); - assertThatCode(() -> ManagerNode.loadNamespaces(List.of(firstStorage, secondStorage), registry, metaStorage, environment)) - .doesNotThrowAnyException(); + assertThatCode( + () -> ManagerNode.loadNamespaces( + List.of(firstStorage, secondStorage), + registry, + metaStorage, + environment)).doesNotThrowAnyException(); verify(registry).createNamespace(firstStorage, metaStorage, environment); verify(registry).createNamespace(secondStorage, metaStorage, environment); @@ -49,25 +52,23 @@ void loadNamespacesReportsOriginalFailureAndAbortsStartup() { ListAppender appender = attachLogAppender(); try { - Throwable thrown = catchThrowable(() -> ManagerNode.loadNamespaces(List.of(storage), registry, metaStorage, environment)); - - assertThat(thrown) - .isInstanceOf(IllegalStateException.class) - .hasMessage("Failed to load 1 of 1 persisted dataset namespaces: [dataset_broken]") - .hasCause(originalFailure); - assertThat(appender.list) - .singleElement() - .satisfies(event -> { - assertThat(event.getLevel()).isEqualTo(Level.ERROR); - assertThat(event.getFormattedMessage()).isEqualTo( - "Failed to load persisted dataset namespace [storage=dataset_broken]. " - + "The dataset is unavailable; application startup will be aborted." - ); - assertThat(event.getThrowableProxy().getClassName()).isEqualTo(IllegalArgumentException.class.getName()); - assertThat(event.getThrowableProxy().getMessage()).isEqualTo("Unknown SQL data source"); - }); - } - finally { + Throwable thrown = catchThrowable( + () -> ManagerNode.loadNamespaces(List.of(storage), registry, metaStorage, environment)); + + assertThat(thrown).isInstanceOf(IllegalStateException.class) + .hasMessage( + "Failed to load 1 of 1 persisted dataset namespaces: [dataset_broken]") + .hasCause(originalFailure); + assertThat(appender.list).singleElement().satisfies(event -> { + assertThat(event.getLevel()).isEqualTo(Level.ERROR); + assertThat(event.getFormattedMessage()).isEqualTo( + "Failed to load persisted dataset namespace [storage=dataset_broken]. " + "The dataset is unavailable; application startup will be aborted." + ); + assertThat(event.getThrowableProxy().getClassName()).isEqualTo( + IllegalArgumentException.class.getName()); + assertThat(event.getThrowableProxy().getMessage()).isEqualTo("Unknown SQL data source"); + }); + } finally { detachLogAppender(appender); } } @@ -92,25 +93,26 @@ void loadNamespacesReportsAllFailures() { ListAppender appender = attachLogAppender(); try { Throwable thrown = catchThrowable( - () -> ManagerNode.loadNamespaces(List.of(firstStorage, workingStorage, secondStorage), registry, metaStorage, environment) + () -> ManagerNode.loadNamespaces( + List.of(firstStorage, workingStorage, secondStorage), + registry, + metaStorage, + environment) ); - assertThat(thrown) - .isInstanceOf(IllegalStateException.class) - .hasMessage("Failed to load 2 of 3 persisted dataset namespaces: [dataset_first, dataset_second]") - .hasCause(firstFailure); + assertThat(thrown).isInstanceOf(IllegalStateException.class) + .hasMessage( + "Failed to load 2 of 3 persisted dataset namespaces: [dataset_first, dataset_second]") + .hasCause( + firstFailure); assertThat(thrown.getSuppressed()).containsExactly(secondFailure); - assertThat(appender.list) - .extracting(ILoggingEvent::getFormattedMessage) - .containsExactly( - "Failed to load persisted dataset namespace [storage=dataset_first]. " - + "The dataset is unavailable; application startup will be aborted.", - "Failed to load persisted dataset namespace [storage=dataset_second]. " - + "The dataset is unavailable; application startup will be aborted." - ); + assertThat(appender.list).extracting(ILoggingEvent::getFormattedMessage) + .containsExactly( + "Failed to load persisted dataset namespace [storage=dataset_first]. " + "The dataset is unavailable; application startup will be aborted.", + "Failed to load persisted dataset namespace [storage=dataset_second]. " + "The dataset is unavailable; application startup will be aborted." + ); verify(registry).createNamespace(workingStorage, metaStorage, environment); - } - finally { + } finally { detachLogAppender(appender); } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java b/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java index a0f9d697912..3f837282d9a 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java @@ -4,10 +4,7 @@ import java.net.URI; import java.util.Collections; -import java.util.List; import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.stream.Stream; import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.apiv1.execution.ResultAsset; @@ -24,14 +21,9 @@ import com.bakdata.conquery.models.exceptions.ValidatorHelper; import com.bakdata.conquery.models.execution.ExecutionState; import com.bakdata.conquery.models.query.DistributedExecutionManager; -import com.bakdata.conquery.models.query.ExecutionManager; import com.bakdata.conquery.models.query.ManagedQuery; -import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.models.query.results.EntityResult; -import com.bakdata.conquery.sql.execution.SqlExecutionExecutionInfo; import com.bakdata.conquery.util.support.StandaloneSupport; import lombok.extern.slf4j.Slf4j; -import org.jetbrains.annotations.NotNull; @Slf4j public class DownloadLinkGeneration extends IntegrationTest.Simple implements ProgrammaticIntegrationTest { @@ -42,7 +34,8 @@ public void execute(StandaloneSupport conquery) throws Exception { final User user = new User("testU", "testU", storage); - final QueryTest test = (QueryTest) ConqueryTestSpec.fromResourcePath("/tests/query/SIMPLE_TREECONCEPT_QUERY/SIMPLE_TREECONCEPT_Query.test.json"); + final QueryTest test = (QueryTest) ConqueryTestSpec.fromResourcePath( + "/tests/query/SIMPLE_TREECONCEPT_QUERY/SIMPLE_TREECONCEPT_Query.test.json"); storage.updateUser(user); @@ -54,7 +47,13 @@ public void execute(StandaloneSupport conquery) throws Exception { Query query = LoadingUtil.parseSubTree(conquery, test.getRawQuery(), Query.class, true); // Create execution for download - ManagedQuery exec = new ManagedQuery(query, user.getId(), conquery.getDataset(), storage, conquery.getDatasetRegistry(), conquery.getConfig()); + ManagedQuery exec = new ManagedQuery( + query, + user.getId(), + conquery.getDataset(), + storage, + conquery.getDatasetRegistry(), + conquery.getConfig()); storage.addExecution(exec); @@ -68,7 +67,8 @@ public void execute(StandaloneSupport conquery) throws Exception { { // Tinker the state of the execution and try again: still not possible because of missing permissions - DistributedExecutionManager.DistributedExecutionInfo distributedState = new DistributedExecutionManager.DistributedExecutionInfo(Collections.emptyList()); + DistributedExecutionManager.DistributedExecutionInfo distributedState = new DistributedExecutionManager.DistributedExecutionInfo( + Collections.emptyList()); distributedState.setExecutionState(ExecutionState.DONE); distributedState.getExecutingLock().countDown(); conquery.getNamespace().getExecutionManager().addState(exec.getId(), distributedState); @@ -83,8 +83,14 @@ public void execute(StandaloneSupport conquery) throws Exception { FullExecutionStatus status = IntegrationUtils.getExecutionStatus(conquery, exec.getId(), user, 200); // This Url is missing the `/api` path part, because we use the standard UriBuilder here - assertThat(status.getResultUrls()).contains(new ResultAsset("CSV", new URI(String.format("%s/result/csv/%s.csv", conquery.defaultApiURIBuilder() - .toString(), exec.getId())))); + assertThat(status.getResultUrls()).contains( + new ResultAsset( + "CSV", + new URI( + String.format( + "%s/result/csv/%s.csv", + conquery.defaultApiURIBuilder().toString(), + exec.getId())))); } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTest.java b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTest.java index 286c247cea5..13825fba065 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTest.java @@ -49,8 +49,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { try { execute(conquery); - } - finally { + } finally { testConquery.removeSupport(conquery); } } @@ -74,21 +73,25 @@ public void execute() throws Throwable { // we clone the default config to ensure that nothing mangles the config of others. // However, override config _should_ not mutate the incoming config. - final ConqueryConfig clonedConfig = Cloner.clone(integrationTests.getConfig(), Map.of(), IntegrationTests.MAPPER); + final ConqueryConfig clonedConfig = Cloner.clone( + integrationTests.getConfig(), + Map.of(), + IntegrationTests.MAPPER); final ConqueryConfig overridenConfig = test.overrideConfig(clonedConfig, integrationTests.getWorkDir()); - final TestConquery testConquery = integrationTests.getCachedConqueryInstance(integrationTests.getWorkDir(), overridenConfig, testImporter); + final TestConquery testConquery = integrationTests.getCachedConqueryInstance( + integrationTests.getWorkDir(), + overridenConfig, + testImporter); try { testConquery.beforeEach(); test.execute(name, testConquery); - } - catch (Exception e) { + } catch (Exception e) { ConqueryMDC.setLocation(name); log.info("FAILED integration test " + name, e); throw e; - } - finally { + } finally { testConquery.afterEach(); } ConqueryMDC.setLocation(name); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java index 8b9fbc2a6e4..e09219c3fcc 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java @@ -17,7 +17,6 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; -import javax.annotation.CheckForNull; import com.bakdata.conquery.TestTags; import com.bakdata.conquery.integration.json.ConqueryTestSpec; @@ -61,9 +60,10 @@ public class IntegrationTests { final ObjectMapper mapper = Jackson.MAPPER.copy(); MAPPER = mapper.setConfig(mapper.getDeserializationConfig().withView(View.Persistence.class)) - .setConfig(mapper.getSerializationConfig().withView(View.Persistence.class)) - // to always deserialize into TestSqlConnectorConfig for our tests - .addMixIn(SqlConnectorConfig.class, TestSqlConnectorConfig.class); + .setConfig( + mapper.getSerializationConfig().withView(View.Persistence.class)) + // to always deserialize into TestSqlConnectorConfig for our tests + .addMixIn(SqlConnectorConfig.class, TestSqlConnectorConfig.class); CONFIG_WRITER = MAPPER.writerFor(ConqueryConfig.class); } @@ -87,38 +87,44 @@ public IntegrationTests(String defaultTestRoot, String defaultTestRootPackage) { private static DynamicContainer toDynamicContainer(ResourceTree currentDir, List list) { list.sort(Comparator.comparing(DynamicNode::getDisplayName)); return dynamicContainer( - currentDir.getName(), - URI.create("classpath:/" + currentDir.getFullName() + "/"), - list.stream() + currentDir.getName(), + URI.create("classpath:/" + currentDir.getFullName() + "/"), + list.stream() ); } private static DynamicTest wrapError(Resource resource, String name, Exception e) { return DynamicTest.dynamicTest( - name, - resource.getURI(), - () -> { - throw e; - } + name, + resource.getURI(), + () -> { + throw e; + } ); } private static ResourceTree scanForResources(String testRoot, String pattern) { ResourceTree tree = new ResourceTree(null, null); - tree.addAll(CPSTypeIdResolver.SCAN_RESULT.getResourcesMatchingPattern(Pattern.compile("^" + testRoot + pattern))); + tree.addAll( + CPSTypeIdResolver.SCAN_RESULT.getResourcesMatchingPattern(Pattern.compile("^" + testRoot + pattern))); return tree; } public List jsonTests() { TestDataImporter testImporter = new WorkerTestDataImporter(); - final String testRoot = Objects.requireNonNullElse(System.getenv(TestTags.TEST_DIRECTORY_ENVIRONMENT_VARIABLE), defaultTestRoot); + final String testRoot = Objects.requireNonNullElse( + System.getenv(TestTags.TEST_DIRECTORY_ENVIRONMENT_VARIABLE), + defaultTestRoot); ResourceTree tree = scanForResources(testRoot, JSON_TEST_PATTERN); Dialect dialect = null; return collectTestTree(tree, testRoot, testImporter, dialect); } @SneakyThrows - public Stream sqlProgrammaticTests(DatabaseConnectionConfig databaseConfig, SqlConnectorConfig sqlConfig, TestDataImporter testDataImporter) { + public Stream sqlProgrammaticTests( + DatabaseConnectionConfig databaseConfig, + SqlConnectorConfig sqlConfig, + TestDataImporter testDataImporter) { this.config.setSqlConnectorConfig(sqlConfig); return programmaticTests(testDataImporter, StandaloneSupport.Mode.SQL); } @@ -126,84 +132,104 @@ public Stream sqlProgrammaticTests(DatabaseConnectionConfig databas @SneakyThrows public Stream programmaticTests(TestDataImporter testImporter, StandaloneSupport.Mode mode) { String regexFilter = System.getenv(TestTags.TEST_PROGRAMMATIC_REGEX_FILTER); - List> programmatic = - CPSTypeIdResolver.SCAN_RESULT.getClassesImplementing(ProgrammaticIntegrationTest.class.getName()) - .filter(info -> info.getPackageName().startsWith(defaultTestRootPackage)) - .filter(classInfo -> { - // e.g. For the RestartTest: CONQUERY_TEST_PROGRAMMATIC_REGEX_FILTER=Restart.* - - if (Strings.isNullOrEmpty(regexFilter)) { - // No filter set: allow all tests - return true; - } - return classInfo.getSimpleName().matches(regexFilter); + List> programmatic = CPSTypeIdResolver.SCAN_RESULT.getClassesImplementing( + ProgrammaticIntegrationTest.class.getName()) + .filter( + info -> info.getPackageName().startsWith(defaultTestRootPackage)) + .filter(classInfo -> { + // e.g. For the RestartTest: CONQUERY_TEST_PROGRAMMATIC_REGEX_FILTER=Restart.* + + if (Strings.isNullOrEmpty(regexFilter)) { + // No filter set: allow all tests + return true; + } + return classInfo.getSimpleName().matches(regexFilter); - }) - .loadClasses(); + }) + .loadClasses(); if (programmatic.isEmpty() && !Strings.isNullOrEmpty(regexFilter)) { throw new RuntimeException("No test cases where found using the filter: " + regexFilter); } - return programmatic - .stream() - .map(c -> { - try { - return c.asSubclass(ProgrammaticIntegrationTest.class).getDeclaredConstructor().newInstance(); - } - catch (Exception e) { - throw new RuntimeException(e); - } - }) - .filter(test -> test.isEnabled(mode)) - .map(programmaticIntegrationTest -> createDynamicProgrammaticTestNode(programmaticIntegrationTest, testImporter)); + return programmatic.stream().map(c -> { + try { + return c.asSubclass(ProgrammaticIntegrationTest.class).getDeclaredConstructor().newInstance(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }) + .filter(test -> test.isEnabled(mode)) + .map( + programmaticIntegrationTest -> createDynamicProgrammaticTestNode( + programmaticIntegrationTest, + testImporter)); } - private DynamicTest createDynamicProgrammaticTestNode(ProgrammaticIntegrationTest test, TestDataImporter testImporter) { + private DynamicTest createDynamicProgrammaticTestNode( + ProgrammaticIntegrationTest test, + TestDataImporter testImporter) { return DynamicTest.dynamicTest( - test.getClass().getSimpleName(), - //classpath URI - URI.create("classpath:/" + test.getClass().getName().replace('.', '/') + ".java"), - new IntegrationTest.Wrapper(test.getClass().getSimpleName(), this, test, testImporter) + test.getClass().getSimpleName(), + //classpath URI + URI.create("classpath:/" + test.getClass().getName().replace('.', '/') + ".java"), + new IntegrationTest.Wrapper(test.getClass().getSimpleName(), this, test, testImporter) ); } @SneakyThrows - public List sqlQueryTests(DatabaseConnectionConfig databaseConfig, SqlConnectorConfig sqlConfig, TestDataImporter testDataImporter) { + public List sqlQueryTests( + DatabaseConnectionConfig databaseConfig, + SqlConnectorConfig sqlConfig, + TestDataImporter testDataImporter) { this.config.setSqlConnectorConfig(sqlConfig); - final String testRoot = Objects.requireNonNullElse(System.getenv(TestTags.SQL_BACKEND_TEST_DIRECTORY_ENVIRONMENT_VARIABLE), defaultTestRoot); + final String testRoot = Objects.requireNonNullElse( + System.getenv(TestTags.SQL_BACKEND_TEST_DIRECTORY_ENVIRONMENT_VARIABLE), + defaultTestRoot); ResourceTree tree = scanForResources(testRoot, SQL_TEST_PATTERN); return collectTestTree(tree, testRoot, testDataImporter, databaseConfig.getDialect()); } - private List collectTestTree(ResourceTree tree, String testRoot, TestDataImporter testImporter, Dialect sqlDialect) { + private List collectTestTree( + ResourceTree tree, + String testRoot, + TestDataImporter testImporter, + Dialect sqlDialect) { if (tree.getChildren().isEmpty()) { throw new RuntimeException("Could not find tests in " + testRoot); } final ResourceTree reduced = tree.reduce(); if (reduced.getChildren().isEmpty()) { - return collectTests(reduced, testImporter, sqlDialect) - .map(List::of) - .orElse(Collections.emptyList()); + return collectTests(reduced, testImporter, sqlDialect).map(List::of).orElse(Collections.emptyList()); } - return reduced.getChildren().values().stream() - .map(currentDir -> collectTests(currentDir, testImporter, sqlDialect)) - .flatMap(Optional::stream) - .collect(Collectors.toList()); + return reduced.getChildren() + .values() + .stream() + .map( + currentDir -> collectTests(currentDir, testImporter, sqlDialect)) + .flatMap(Optional::stream) + .collect( + Collectors.toList()); } - private Optional collectTests(ResourceTree currentDir, TestDataImporter testImporter, Dialect sqlDialect) { + private Optional collectTests( + ResourceTree currentDir, + TestDataImporter testImporter, + Dialect sqlDialect) { if (currentDir.getValue() != null) { - Optional dynamicTest = readTest(currentDir.getValue(), currentDir.getName(), testImporter, sqlDialect); + Optional dynamicTest = readTest( + currentDir.getValue(), + currentDir.getName(), + testImporter, + sqlDialect); if (dynamicTest.isPresent()) { return Optional.of(dynamicTest.get()); } } List list = new ArrayList<>(); for (ResourceTree child : currentDir.getChildren().values()) { - collectTests(child, testImporter, sqlDialect) - .ifPresent(list::add); + collectTests(child, testImporter, sqlDialect).ifPresent(list::add); } if (list.isEmpty()) { @@ -213,7 +239,11 @@ private Optional collectTests(ResourceTree currentDir, TestDataImpo return Optional.of(toDynamicContainer(currentDir, list)); } - private Optional readTest(Resource resource, String name, TestDataImporter testImporter, Dialect sqlDialect) { + private Optional readTest( + Resource resource, + String name, + TestDataImporter testImporter, + Dialect sqlDialect) { try { ConqueryTestSpec conqueryTestSpec = ConqueryTestSpec.fromResource(resource); JsonIntegrationTest test = new JsonIntegrationTest(conqueryTestSpec); @@ -223,37 +253,44 @@ private Optional readTest(Resource resource, String name, TestDataI } return Optional.of(wrapTest(resource, name, test, testImporter)); - } - catch (Exception e) { + } catch (Exception e) { return Optional.of(wrapError(resource, name, e)); } } - private DynamicTest wrapTest(Resource resource, String name, JsonIntegrationTest test, TestDataImporter testImporter) { + private DynamicTest wrapTest( + Resource resource, + String name, + JsonIntegrationTest test, + TestDataImporter testImporter) { String testLabel = Optional.ofNullable(test.getTestSpec().getLabel()) - // If no label was defined use the filename part before the first dot - .orElse(name.split("\\.", 1)[0]); + // If no label was defined use the filename part before the first dot + .orElse(name.split("\\.", 1)[0]); // For easier modification we link the source- not the target-resource of the json tests. // Otherwise, a modification might affect the current test runs, // but it won't persist over rebuilds or in version control final URI compileTargetURI = resource.getURI(); - final URI sourceResourceURI = URI.create(compileTargetURI.toString().replace("/target/test-classes/", "/src/test/resources/")); + final URI sourceResourceURI = URI.create( + compileTargetURI.toString().replace("/target/test-classes/", "/src/test/resources/")); return DynamicTest.dynamicTest( + testLabel, + sourceResourceURI, + new IntegrationTest.Wrapper( testLabel, - sourceResourceURI, - new IntegrationTest.Wrapper( - testLabel, - this, - test, - testImporter - ) + this, + test, + testImporter + ) ); } @SneakyThrows - public synchronized TestConquery getCachedConqueryInstance(File workDir, ConqueryConfig conf, TestDataImporter testDataImporter) { + public synchronized TestConquery getCachedConqueryInstance( + File workDir, + ConqueryConfig conf, + TestDataImporter testDataImporter) { // This should be fast enough and a stable comparison String confString = CONFIG_WRITER.writeValueAsString(conf); if (!reusedInstances.containsKey(confString)) { @@ -263,7 +300,9 @@ public synchronized TestConquery getCachedConqueryInstance(File workDir, Conquer ConfigOverride.configureRandomPorts(conf); if (conf.getStorage() instanceof XodusStoreFactory storeFactory) { - ConfigOverride.configureWorkdir(storeFactory, workDir.toPath().resolve(String.valueOf(confString.hashCode()))); + ConfigOverride.configureWorkdir( + storeFactory, + workDir.toPath().resolve(String.valueOf(confString.hashCode()))); } log.trace("Creating a new test conquery instance for test {}", conf); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/ResourceTree.java b/backend/src/test/java/com/bakdata/conquery/integration/ResourceTree.java index d276009d0f1..13796c2b7da 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/ResourceTree.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/ResourceTree.java @@ -16,7 +16,9 @@ import lombok.ToString; import org.apache.commons.lang3.StringUtils; -@RequiredArgsConstructor @Getter @ToString +@RequiredArgsConstructor +@Getter +@ToString public class ResourceTree { @ToString.Include private final String name; @@ -29,7 +31,7 @@ public class ResourceTree { private Map children = new HashMap<>(); public void addAll(Iterable resources) { - for(Resource r : resources) { + for (Resource r : resources) { add(r); } } @@ -40,19 +42,17 @@ public void add(Resource r) { } private void add(Resource r, String[] parts, int index) { - if(index == parts.length) { + if (index == parts.length) { this.value = r; return; } - children - .computeIfAbsent(parts[index], name -> new ResourceTree(name, this)) - .add(r, parts, index+1); + children.computeIfAbsent(parts[index], name -> new ResourceTree(name, this)).add(r, parts, index + 1); } public String getFullName() { - List l = Stream - .iterate(this, Objects::nonNull, ResourceTree::getParent) - .map(ResourceTree::getName) + List l = Stream.iterate(this, Objects::nonNull, ResourceTree::getParent) + .map( + ResourceTree::getName) .filter(Objects::nonNull) .collect(Collectors.toList()); Collections.reverse(l); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/common/AutoConceptUtil.java b/backend/src/test/java/com/bakdata/conquery/integration/common/AutoConceptUtil.java index f77a4e3c594..e440758cf16 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/common/AutoConceptUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/common/AutoConceptUtil.java @@ -98,9 +98,9 @@ private static List

tables = conquery.getNamespace().getStorage().getTables()) { @@ -129,9 +129,8 @@ public void execute(String name, TestConquery testConquery) throws Exception { assertThat(namespace.getStorage().getAllImports().count()).isEqualTo(0); // The deleted import should not be found. - assertThat(namespace.getStorage().getAllImports()) - .filteredOn(imp -> imp.getTable().getDataset().equals(dataset)) - .isEmpty(); + assertThat(namespace.getStorage().getAllImports()).filteredOn( + imp -> imp.getTable().getDataset().equals(dataset)).isEmpty(); for (ShardNode node : conquery.getShardNodes()) { for (Worker value : node.getWorkers().getWorkers().values()) { @@ -143,7 +142,13 @@ public void execute(String name, TestConquery testConquery) throws Exception { // Try to execute the query after deletion - IntegrationUtils.assertQueryResult(conquery, query, 0, ExecutionState.FAILED, conquery.getTestUser(), 404); + IntegrationUtils.assertQueryResult( + conquery, + query, + 0, + ExecutionState.FAILED, + conquery.getTestUser(), + 404); } } @@ -165,26 +170,36 @@ public void execute(String name, TestConquery testConquery) throws Exception { LoadingUtil.importConcepts(conqueryReimport, test.getRawConcepts()); conqueryReimport.waitUntilWorkDone(); - assertThat(conqueryReimport.getAdminDatasetsProcessor().getDatasetRegistry().get(conqueryReimport.getDataset())) - .describedAs("Dataset after re-import.") - .isNotNull(); + assertThat( + conqueryReimport.getAdminDatasetsProcessor() + .getDatasetRegistry() + .get( + conqueryReimport.getDataset())).describedAs("Dataset after re-import.").isNotNull(); try (Stream allImports = conqueryReimport.getNamespace().getStorage().getAllImports()) { assertThat(allImports.count()).isEqualTo(nImports); } for (ShardNode node : conqueryReimport.getShardNodes()) { - assertThat(node.getWorkers().getWorkers().values()) - .filteredOn(w -> w.getInfo().getDataset().equals(conqueryReimport.getDataset())) - .describedAs("Workers for node {}", node.getName()) - .isNotEmpty(); + assertThat(node.getWorkers().getWorkers().values()).filteredOn( + w -> w.getInfo().getDataset().equals(conqueryReimport.getDataset())) + .describedAs( + "Workers for node {}", + node.getName()) + .isNotEmpty(); } log.info("Executing query after re-import"); final Query query2 = IntegrationUtils.parseQuery(conqueryReimport, test.getRawQuery()); // Issue a query and assert that it has the same content as the first time around. - IntegrationUtils.assertQueryResult(conqueryReimport, query2, 2L, ExecutionState.DONE, conqueryReimport.getTestUser(), 201); + IntegrationUtils.assertQueryResult( + conqueryReimport, + query2, + 2L, + ExecutionState.DONE, + conqueryReimport.getTestUser(), + 201); } @@ -210,9 +225,12 @@ public void execute(String name, TestConquery testConquery) throws Exception { final ModificationShieldedWorkerStorage workerStorage = value.getStorage(); - assertThat(IntegrationUtils.getAllBuckets(workerStorage).filter(bucket -> bucket.getTable().getDataset().equals(dataset))) - .describedAs("Buckets for Worker %s", value.getInfo().getId()) - .isNotEmpty(); + assertThat( + IntegrationUtils.getAllBuckets(workerStorage) + .filter( + bucket -> bucket.getTable().getDataset().equals(dataset))).describedAs( + "Buckets for Worker %s", + value.getInfo().getId()).isNotEmpty(); } } @@ -220,9 +238,14 @@ public void execute(String name, TestConquery testConquery) throws Exception { final Query query3 = IntegrationUtils.parseQuery(conqueryRestart, test.getRawQuery()); // Issue a query and assert that it has the same content as the first time around. - IntegrationUtils.assertQueryResult(conqueryRestart, query3, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult( + conqueryRestart, + query3, + 2L, + ExecutionState.DONE, + conquery.getTestUser(), + 201); } } } } - diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java index dc4448a7c19..2122ff6e024 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java @@ -3,6 +3,8 @@ import static com.bakdata.conquery.integration.common.LoadingUtil.importSecondaryIds; import static org.assertj.core.api.Assertions.assertThat; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; import java.io.File; import java.io.FileInputStream; import java.net.URI; @@ -10,8 +12,6 @@ import java.util.Map; import java.util.stream.Stream; import java.util.zip.GZIPInputStream; -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.Response; import com.bakdata.conquery.ConqueryConstants; import com.bakdata.conquery.apiv1.query.Query; @@ -58,7 +58,8 @@ public void execute(String name, TestConquery testConquery) throws Exception { final StandaloneSupport conquery = testConquery.getSupport(name); - final String testJson = LoadingUtil.readResource("/tests/query/DELETE_IMPORT_TESTS/SIMPLE_TREECONCEPT_Query.test.json"); + final String testJson = LoadingUtil.readResource( + "/tests/query/DELETE_IMPORT_TESTS/SIMPLE_TREECONCEPT_Query.test.json"); final DatasetId dataset = conquery.getDataset(); final Namespace namespace = conquery.getNamespace(); @@ -99,13 +100,10 @@ public void execute(String name, TestConquery testConquery) throws Exception { // Must contain the import. try (Stream allImports = namespace.getStorage().getAllImports()) { - assertThat(allImports) - .filteredOn(imp -> imp.equals(importId)) - .isNotEmpty(); + assertThat(allImports).filteredOn(imp -> imp.equals(importId)).isNotEmpty(); } - assertThat(namespace.getStorage().getImport(importId)) - .isNotNull(); + assertThat(namespace.getStorage().getImport(importId)).isNotNull(); for (ShardNode node : conquery.getShardNodes()) { for (Worker worker : node.getWorkers().getWorkers().values()) { @@ -115,18 +113,21 @@ public void execute(String name, TestConquery testConquery) throws Exception { final ModificationShieldedWorkerStorage workerStorage = worker.getStorage(); - assertThat(workerStorage.getAllCBlocks()) - .describedAs("CBlocks for Worker %s", worker.getInfo().getId()) - .filteredOn(block -> block.getBucket().getDataset().equals(dataset)) - .isNotEmpty(); - assertThat(IntegrationUtils.getAllBuckets(workerStorage)) - .filteredOn(bucket -> bucket.getId().getDataset().equals(dataset)) - .describedAs("Buckets for Worker %s", worker.getInfo().getId()) - .isNotEmpty(); + assertThat(workerStorage.getAllCBlocks()).describedAs( + "CBlocks for Worker %s", + worker.getInfo().getId()) + .filteredOn( + block -> block.getBucket().getDataset().equals(dataset)) + .isNotEmpty(); + assertThat(IntegrationUtils.getAllBuckets(workerStorage)).filteredOn( + bucket -> bucket.getId().getDataset().equals(dataset)) + .describedAs( + "Buckets for Worker %s", + worker.getInfo().getId()) + .isNotEmpty(); // Must contain the import. - assertThat(workerStorage.getImport(importId)) - .isNotNull(); + assertThat(workerStorage.getImport(importId)).isNotNull(); } } @@ -139,15 +140,25 @@ public void execute(String name, TestConquery testConquery) throws Exception { { log.info("Issuing deletion of import {}", importId); - final URI deleteImportUri = - HierarchyHelper.hierarchicalPath(conquery.defaultAdminURIBuilder(), AdminTablesResource.class, "deleteImport") - .buildFromMap(Map.of( - ResourceConstants.DATASET, conquery.getDataset(), - ResourceConstants.TABLE, importId.getTable(), - ResourceConstants.IMPORT_ID, importId - )); - - final Response delete = conquery.getClient().target(deleteImportUri).request(MediaType.APPLICATION_JSON).delete(); + final URI deleteImportUri = HierarchyHelper.hierarchicalPath( + conquery.defaultAdminURIBuilder(), + AdminTablesResource.class, + "deleteImport") + .buildFromMap( + Map.of( + ResourceConstants.DATASET, + conquery.getDataset(), + ResourceConstants.TABLE, + importId.getTable(), + ResourceConstants.IMPORT_ID, + importId + )); + + final Response delete = conquery.getClient() + .target(deleteImportUri) + .request( + MediaType.APPLICATION_JSON) + .delete(); assertThat(delete.getStatusInfo().getFamily()).isEqualTo(Response.Status.Family.SUCCESSFUL); @@ -164,9 +175,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { assertThat(imports.size()).isEqualTo(nImports - 1); // The deleted import should not be found. - assertThat(imports) - .filteredOn(imp -> imp.equals(importId)) - .isEmpty(); + assertThat(imports).filteredOn(imp -> imp.equals(importId)).isEmpty(); } for (ShardNode node : conquery.getShardNodes()) { @@ -178,21 +187,22 @@ public void execute(String name, TestConquery testConquery) throws Exception { final ModificationShieldedWorkerStorage workerStorage = worker.getStorage(); // No bucket should be found referencing the import. - assertThat(IntegrationUtils.getAllBuckets(workerStorage)) - .describedAs("Buckets for Worker %s", worker.getInfo().getId()) - .filteredOn(bucket -> bucket.getImp().equals(importId)) - .isEmpty(); + assertThat(IntegrationUtils.getAllBuckets(workerStorage)).describedAs( + "Buckets for Worker %s", + worker.getInfo().getId()).filteredOn(bucket -> bucket.getImp().equals(importId)).isEmpty(); // No CBlock associated with import may exist - assertThat(workerStorage.getAllCBlocks()) - .describedAs("CBlocks for Worker %s", worker.getInfo().getId()) - .filteredOn(cBlock -> cBlock.getBucket().getImp().equals(importId)) - .isEmpty(); + assertThat(workerStorage.getAllCBlocks()).describedAs( + "CBlocks for Worker %s", + worker.getInfo().getId()) + .filteredOn( + cBlock -> cBlock.getBucket().getImp().equals(importId)) + .isEmpty(); // Import should not exists anymore - assertThat(workerStorage.getImport(importId)) - .describedAs("Import for Worker %s", worker.getInfo().getId()) - .isNull(); + assertThat(workerStorage.getImport(importId)).describedAs( + "Import for Worker %s", + worker.getInfo().getId()).isNull(); } } @@ -208,10 +218,12 @@ public void execute(String name, TestConquery testConquery) throws Exception { // Load more data under the same name into the same table, with only the deleted import/table { // only import the deleted import/table - final RequiredTable import2Table = content.getTables().stream() - .filter(table -> table.getName().equalsIgnoreCase(importId.getTable().getTable())) - .findFirst() - .orElseThrow(); + final RequiredTable import2Table = content.getTables() + .stream() + .filter( + table -> table.getName().equalsIgnoreCase(importId.getTable().getTable())) + .findFirst() + .orElseThrow(); final ResourceFile csv = import2Table.getCsv(); @@ -219,12 +231,17 @@ public void execute(String name, TestConquery testConquery) throws Exception { //copy csv to tmp folder // Content 2.2 contains an extra entry of a value that hasn't been seen before. - FileUtils.copyInputStreamToFile(LoadingUtil.openResource(path.substring(0, path.lastIndexOf('/')) + "/" + "content2.2.csv"), - new File(conquery.getTmpDir(), csv.getName()) + FileUtils.copyInputStreamToFile( + LoadingUtil.openResource(path.substring(0, path.lastIndexOf('/')) + "/" + "content2.2.csv"), + new File(conquery.getTmpDir(), csv.getName()) ); - File descriptionFile = new File(conquery.getTmpDir(), import2Table.getName() + ConqueryConstants.EXTENSION_DESCRIPTION); - File preprocessedFile = new File(conquery.getTmpDir(), import2Table.getName() + ConqueryConstants.EXTENSION_PREPROCESSED); + File descriptionFile = new File( + conquery.getTmpDir(), + import2Table.getName() + ConqueryConstants.EXTENSION_DESCRIPTION); + File preprocessedFile = new File( + conquery.getTmpDir(), + import2Table.getName() + ConqueryConstants.EXTENSION_PREPROCESSED); //create import descriptor @@ -247,7 +264,10 @@ public void execute(String name, TestConquery testConquery) throws Exception { conquery.preprocessTmp(conquery.getTmpDir(), List.of(descriptionFile)); //import preprocessedFiles - conquery.getAdminDatasetsProcessor().addImport(conquery.getNamespace(), new GZIPInputStream(new FileInputStream(preprocessedFile))); + conquery.getAdminDatasetsProcessor() + .addImport( + conquery.getNamespace(), + new GZIPInputStream(new FileInputStream(preprocessedFile))); conquery.waitUntilWorkDone(); } @@ -267,11 +287,13 @@ public void execute(String name, TestConquery testConquery) throws Exception { final ModificationShieldedWorkerStorage workerStorage = worker.getStorage(); - assertThat(IntegrationUtils.getAllBuckets(workerStorage)) - .describedAs("Buckets for Worker %s", worker.getInfo().getId()) - .filteredOn(bucket -> bucket.getImp().equals(importId)) - .filteredOn(bucket -> bucket.getId().getDataset().equals(dataset)) - .isNotEmpty(); + assertThat(IntegrationUtils.getAllBuckets(workerStorage)).describedAs( + "Buckets for Worker %s", + worker.getInfo().getId()) + .filteredOn(bucket -> bucket.getImp().equals(importId)) + .filteredOn( + bucket -> bucket.getId().getDataset().equals(dataset)) + .isNotEmpty(); } } @@ -305,11 +327,14 @@ public void execute(String name, TestConquery testConquery) throws Exception { final ModificationShieldedWorkerStorage workerStorage = worker.getStorage(); try (Stream allBuckets = IntegrationUtils.getAllBuckets(workerStorage)) { - assertThat(allBuckets) - .describedAs("Buckets for Worker %s", worker.getInfo().getId()) - .filteredOn(bucket -> bucket.getId().getDataset().equals(dataset)) - .filteredOn(bucket -> bucket.getImp().equals(importId)) - .isNotEmpty(); + assertThat(allBuckets).describedAs( + "Buckets for Worker %s", + worker.getInfo().getId()) + .filteredOn( + bucket -> bucket.getId().getDataset().equals(dataset)) + .filteredOn( + bucket -> bucket.getImp().equals(importId)) + .isNotEmpty(); } } } @@ -317,7 +342,13 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query after re-import"); // Issue a query and assert that it has the same content as the first time around. - IntegrationUtils.assertQueryResult(conquery2, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult( + conquery2, + query, + 2L, + ExecutionState.DONE, + conquery.getTestUser(), + 201); } } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/TableDeletionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/TableDeletionTest.java index bf70a1f8fa3..51c7bb9e3c6 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/TableDeletionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/TableDeletionTest.java @@ -3,11 +3,11 @@ import static com.bakdata.conquery.integration.common.LoadingUtil.importSecondaryIds; import static org.assertj.core.api.Assertions.assertThat; +import jakarta.ws.rs.core.Response; import java.net.URI; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; -import jakarta.ws.rs.core.Response; import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.commands.ShardNode; @@ -46,7 +46,8 @@ public void execute(String name, TestConquery testConquery) throws Exception { final StandaloneSupport conquery = testConquery.getSupport(name); - final String testJson = LoadingUtil.readResource("/tests/query/DELETE_IMPORT_TESTS/SIMPLE_TREECONCEPT_Query.test.json"); + final String testJson = LoadingUtil.readResource( + "/tests/query/DELETE_IMPORT_TESTS/SIMPLE_TREECONCEPT_Query.test.json"); final DatasetId dataset = conquery.getDataset(); final Namespace namespace = conquery.getNamespace(); @@ -77,7 +78,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { final long nImports; try (Stream allImports = namespace.getStorage().getAllImports()) { - nImports = allImports.count(); + nImports = allImports.count(); } @@ -85,8 +86,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { { log.info("Checking state before deletion"); // Must contain the import. - assertThat(namespace.getStorage().getTable(tableId)) - .isNotNull(); + assertThat(namespace.getStorage().getTable(tableId)).isNotNull(); for (ShardNode node : conquery.getShardNodes()) { for (Worker value : node.getWorkers().getWorkers().values()) { @@ -96,12 +96,12 @@ public void execute(String name, TestConquery testConquery) throws Exception { final ModificationShieldedWorkerStorage workerStorage = value.getStorage(); - assertThat(workerStorage.getAllCBlocks()) - .describedAs("CBlocks for Worker %s", value.getInfo().getId()) - .isNotEmpty(); - assertThat(IntegrationUtils.getAllBuckets(workerStorage)) - .describedAs("Buckets for Worker %s", value.getInfo().getId()) - .isNotEmpty(); + assertThat(workerStorage.getAllCBlocks()).describedAs( + "CBlocks for Worker %s", + value.getInfo().getId()).isNotEmpty(); + assertThat(IntegrationUtils.getAllBuckets(workerStorage)).describedAs( + "Buckets for Worker %s", + value.getInfo().getId()).isNotEmpty(); } } @@ -117,21 +117,23 @@ public void execute(String name, TestConquery testConquery) throws Exception { // Delete the import via API. // But, we do not allow deletion of tables with associated connectors, so this should throw! - final URI deleteTable = - HierarchyHelper.hierarchicalPath(conquery.defaultAdminURIBuilder(), AdminTablesResource.class, "remove") - .buildFromMap(Map.of( - ResourceConstants.DATASET, conquery.getDataset().getName(), - ResourceConstants.TABLE, tableId.toString() - )); + final URI deleteTable = HierarchyHelper.hierarchicalPath( + conquery.defaultAdminURIBuilder(), + AdminTablesResource.class, + "remove") + .buildFromMap( + Map.of( + ResourceConstants.DATASET, + conquery.getDataset().getName(), + ResourceConstants.TABLE, + tableId.toString() + )); - final Response failed = conquery.getClient() - .target(deleteTable) - .request() - .delete(); + final Response failed = conquery.getClient().target(deleteTable).request().delete(); assertThat(failed.getStatusInfo().getFamily()).isEqualTo(Response.Status.Family.CLIENT_ERROR); - try(Stream> allConcepts = conquery.getNamespace().getStorage().getAllConcepts()) { + try (Stream> allConcepts = conquery.getNamespace().getStorage().getAllConcepts()) { conquery.getAdminDatasetsProcessor().deleteConcept(allConcepts.iterator().next().getId()); } @@ -160,9 +162,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { // The deleted import should not be found. try (Stream allImports = namespace.getStorage().getAllImports()) { - assertThat(allImports) - .filteredOn(imp -> imp.getTable().equals(tableId)) - .isEmpty(); + assertThat(allImports).filteredOn(imp -> imp.getTable().equals(tableId)).isEmpty(); } for (ShardNode node : conquery.getShardNodes()) { @@ -176,18 +176,18 @@ public void execute(String name, TestConquery testConquery) throws Exception { // No bucket should be found referencing the import. try (Stream allBuckets = IntegrationUtils.getAllBuckets(workerStorage)) { - assertThat(allBuckets) - .describedAs("Buckets for Worker %s", value.getInfo().getId()) - .filteredOn(bucket -> bucket.getImp().getTable().equals(tableId)) - .isEmpty(); + assertThat(allBuckets).describedAs("Buckets for Worker %s", value.getInfo().getId()) + .filteredOn( + bucket -> bucket.getImp().getTable().equals(tableId)) + .isEmpty(); } // No CBlock associated with import may exist - try(Stream allCBlocks = workerStorage.getAllCBlocks()) { - assertThat(allCBlocks) - .describedAs("CBlocks for Worker %s", value.getInfo().getId()) - .filteredOn(cBlock -> cBlock.getBucket().getImp().getTable().equals(tableId)) - .isEmpty(); + try (Stream allCBlocks = workerStorage.getAllCBlocks()) { + assertThat(allCBlocks).describedAs("CBlocks for Worker %s", value.getInfo().getId()) + .filteredOn( + cBlock -> cBlock.getBucket().getImp().getTable().equals(tableId)) + .isEmpty(); } } @@ -203,23 +203,30 @@ public void execute(String name, TestConquery testConquery) throws Exception { // Load the same import into the same table, with only the deleted import/table { // only import the deleted import/table - LoadingUtil.importTables(conquery, content.getTables().stream() - .filter(table -> table.getName().equalsIgnoreCase(tableId.getTable())) - .collect(Collectors.toList()), content.isAutoConcept()); + LoadingUtil.importTables( + conquery, + content.getTables() + .stream() + .filter( + table -> table.getName().equalsIgnoreCase(tableId.getTable())) + .collect(Collectors.toList()), + content.isAutoConcept()); conquery.waitUntilWorkDone(); - LoadingUtil.importTableContents(conquery, content.getTables().stream() - .filter(table -> table.getName().equalsIgnoreCase(tableId.getTable())) - .collect(Collectors.toList())); + LoadingUtil.importTableContents( + conquery, + content.getTables() + .stream() + .filter( + table -> table.getName().equalsIgnoreCase(tableId.getTable())) + .collect(Collectors.toList())); conquery.waitUntilWorkDone(); LoadingUtil.importConcepts(conquery, test.getRawConcepts()); conquery.waitUntilWorkDone(); - assertThat(namespace.getStorage().getTable(tableId)) - .describedAs("Table after re-import.") - .isNotNull(); + assertThat(namespace.getStorage().getTable(tableId)).describedAs("Table after re-import.").isNotNull(); for (ShardNode node : conquery.getShardNodes()) { for (Worker value : node.getWorkers().getWorkers().values()) { @@ -227,9 +234,8 @@ public void execute(String name, TestConquery testConquery) throws Exception { continue; } - assertThat(value.getStorage().getTable(tableId)) - .describedAs("Table in worker storage.") - .isNotNull(); + assertThat(value.getStorage().getTable(tableId)).describedAs( + "Table in worker storage.").isNotNull(); } } } @@ -250,9 +256,9 @@ public void execute(String name, TestConquery testConquery) throws Exception { final ModificationShieldedWorkerStorage workerStorage = value.getStorage(); try (Stream allBuckets = IntegrationUtils.getAllBuckets(workerStorage)) { - assertThat(allBuckets.filter(bucket -> bucket.getImp().getTable().equals(tableId))) - .describedAs("Buckets for Worker %s", value.getInfo().getId()) - .isNotEmpty(); + assertThat(allBuckets.filter(bucket -> bucket.getImp().getTable().equals(tableId))).describedAs( + "Buckets for Worker %s", + value.getInfo().getId()).isNotEmpty(); } } } @@ -289,9 +295,10 @@ public void execute(String name, TestConquery testConquery) throws Exception { final ModificationShieldedWorkerStorage workerStorage = value.getStorage(); try (Stream allBuckets = IntegrationUtils.getAllBuckets(workerStorage)) { - assertThat(allBuckets.filter(bucket -> bucket.getImp().getTable().equals(tableId))) - .describedAs("Buckets for Worker %s", value.getInfo().getId()) - .isNotEmpty(); + assertThat( + allBuckets.filter(bucket -> bucket.getImp().getTable().equals(tableId))).describedAs( + "Buckets for Worker %s", + value.getInfo().getId()).isNotEmpty(); } } @@ -300,7 +307,13 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Executing query after re-import and restart"); // Issue a query and assert that it has the same content as the first time around. - IntegrationUtils.assertQueryResult(conquery2, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); + IntegrationUtils.assertQueryResult( + conquery2, + query, + 2L, + ExecutionState.DONE, + conquery.getTestUser(), + 201); } } } diff --git a/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java b/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java index 80c0b3a5ff5..48af5cdcd03 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java @@ -50,7 +50,10 @@ public void before() { namespaceStorage = storageFactory.createNamespaceStorage(); workerStorage = storageFactory.createWorkerStorage(); - final ClusterNamespaceHandler clusterNamespaceHandler = new ClusterNamespaceHandler(new ClusterState(), config, internalMapperFactory); + final ClusterNamespaceHandler clusterNamespaceHandler = new ClusterNamespaceHandler( + new ClusterState(), + config, + internalMapperFactory); datasetRegistry = new DatasetRegistry<>(config, internalMapperFactory, clusterNamespaceHandler, indexService) { @Override public NamespacedStorage getStorage(DatasetId datasetId) { @@ -61,7 +64,9 @@ public NamespacedStorage getStorage(DatasetId datasetId) { managerInternalMapper = internalMapperFactory.createManagerPersistenceMapper(datasetRegistry, metaStorage); metaStorage.openStores(managerInternalMapper); - namespacePersistenceMapper = internalMapperFactory.createNamespacePersistenceMapper(namespaceStorage, datasetRegistry); + namespacePersistenceMapper = internalMapperFactory.createNamespacePersistenceMapper( + namespaceStorage, + datasetRegistry); namespaceStorage.openStores(namespacePersistenceMapper); shardNamespacedStorageProvider = new TestNamespacedStorageProvider(getWorkerStorage()); diff --git a/backend/src/test/java/com/bakdata/conquery/io/cps/CPSBaseTest.java b/backend/src/test/java/com/bakdata/conquery/io/cps/CPSBaseTest.java index 702e6820f77..aab5eb1ce00 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/cps/CPSBaseTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/cps/CPSBaseTest.java @@ -13,36 +13,35 @@ public class CPSBaseTest { @TestFactory public Stream testCPSBase() { - return CPSTypeIdResolver - .listImplementations() + return CPSTypeIdResolver.listImplementations() .stream() .map(this::createTest) - .sorted(Comparator.comparing(DynamicTest::getDisplayName)); + .sorted( + Comparator.comparing(DynamicTest::getDisplayName)); } - - public DynamicTest createTest(Pair,Class> baseTypePair) { + + public DynamicTest createTest(Pair, Class> baseTypePair) { String name; - if(baseTypePair.getRight().getAnnotation(CPSType.class)!=null) { + if (baseTypePair.getRight().getAnnotation(CPSType.class) != null) { name = baseTypePair.getRight().getAnnotation(CPSType.class).id(); - } - else { + } else { name = "multiple"; } - - name += " -> "+baseTypePair.getRight().getSimpleName(); - return DynamicTest.dynamicTest(name, ()->test(baseTypePair.getLeft(), baseTypePair.getRight())); + + name += " -> " + baseTypePair.getRight().getSimpleName(); + return DynamicTest.dynamicTest(name, () -> test(baseTypePair.getLeft(), baseTypePair.getRight())); } public void test(Class base, Class type) { CPSType[] annos = type.getAnnotationsByType(CPSType.class); assertThat(annos).isNotEmpty(); - for(CPSType anno : annos) { + for (CPSType anno : annos) { assertThat(anno.base()).isNotNull(); assertThat(anno.id()).isNotEmpty(); assertThat(anno.base()).isEqualTo(base); assertThat(base).hasAnnotation(CPSBase.class); assertThat(base).isAssignableFrom(base); - if(anno.subTyped()) { + if (anno.subTyped()) { assertThat(SubTyped.class).isAssignableFrom(type); } } diff --git a/backend/src/test/java/com/bakdata/conquery/io/jackson/JacksonTest.java b/backend/src/test/java/com/bakdata/conquery/io/jackson/JacksonTest.java index e023d2f32e3..1dacd53b1be 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/jackson/JacksonTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/jackson/JacksonTest.java @@ -20,27 +20,32 @@ public class JacksonTest { @Test public void testSingleElementArraySerialization() throws JsonProcessingException { - String[] arr = new String[] {"singular"}; - assertThat(Jackson.MAPPER.writeValueAsString(arr)) - .isEqualTo("[\"singular\"]"); + String[] arr = new String[]{"singular"}; + assertThat(Jackson.MAPPER.writeValueAsString(arr)).isEqualTo("[\"singular\"]"); } - + @Test public void testSingleElementListSerialization() throws JsonProcessingException { List list = Arrays.asList("singular"); - assertThat(Jackson.MAPPER.writeValueAsString(list)) - .isEqualTo("[\"singular\"]"); + assertThat(Jackson.MAPPER.writeValueAsString(list)).isEqualTo("[\"singular\"]"); } public static Stream arguments() { - return Stream - .of( - Arguments.of(null, "{\"external\":0,\"apiPersistent\":1,\"internalOnly\":2,\"internalCommunication\":3,\"persistentManager\":5,\"persistentShard\":6,\"persistent\":7,\"api\":8}"), - Arguments.of(View.InternalCommunication.class, "{\"external\":0,\"internalOnly\":2,\"internalCommunication\":3}"), - Arguments.of(View.Persistence.Manager.class, "{\"external\":0,\"apiPersistent\":1,\"internalOnly\":2,\"persistentManager\":5,\"persistent\":7}"), - Arguments.of(View.Persistence.Shard.class, "{\"external\":0,\"internalOnly\":2,\"persistentShard\":6,\"persistent\":7}"), - Arguments.of(View.Api.class, "{\"external\":0,\"apiPersistent\":1,\"api\":8}") - ); + return Stream.of( + Arguments.of( + null, + "{\"external\":0,\"apiPersistent\":1,\"internalOnly\":2,\"internalCommunication\":3,\"persistentManager\":5,\"persistentShard\":6,\"persistent\":7,\"api\":8}"), + Arguments.of( + View.InternalCommunication.class, + "{\"external\":0,\"internalOnly\":2,\"internalCommunication\":3}"), + Arguments.of( + View.Persistence.Manager.class, + "{\"external\":0,\"apiPersistent\":1,\"internalOnly\":2,\"persistentManager\":5,\"persistent\":7}"), + Arguments.of( + View.Persistence.Shard.class, + "{\"external\":0,\"internalOnly\":2,\"persistentShard\":6,\"persistent\":7}"), + Arguments.of(View.Api.class, "{\"external\":0,\"apiPersistent\":1,\"api\":8}") + ); } @ParameterizedTest @@ -51,10 +56,9 @@ public void testViews(Class viewClass, String expected) throws J if (viewClass != null) { writer = writer.withView(viewClass); } - assertThat(writer.writeValueAsString(test)) - .isEqualTo(expected); + assertThat(writer.writeValueAsString(test)).isEqualTo(expected); } - + @Data public static class InternalTestClass { @@ -81,6 +85,7 @@ public static class InternalTestClass { @JsonView(View.Api.class) private int api = 8; } - - public static class Marker {} + + public static class Marker { + } } diff --git a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/ClassToInstanceMapDeserializerTest.java b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/ClassToInstanceMapDeserializerTest.java index 09c14ecf20a..b09d595285e 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/ClassToInstanceMapDeserializerTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/ClassToInstanceMapDeserializerTest.java @@ -18,8 +18,8 @@ void test() throws JSONException, IOException { ClassToInstanceMap map = MutableClassToInstanceMap.create(); map.putInstance(MajorTypeId.class, MajorTypeId.DATE); map.putInstance(String.class, "test"); - - String v=Jackson.MAPPER.writeValueAsString(map); + + String v = Jackson.MAPPER.writeValueAsString(map); assertThat( Jackson.MAPPER.readValue(v, ClassToInstanceMap.class) ).isEqualTo(map); diff --git a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java index c40efe10a23..2ff340127cb 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java @@ -3,6 +3,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; +import jakarta.validation.Validator; import java.io.IOException; import java.lang.ref.SoftReference; import java.lang.ref.WeakReference; @@ -12,7 +13,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.UnaryOperator; -import jakarta.validation.Validator; import com.bakdata.conquery.io.jackson.Injectable; import com.bakdata.conquery.io.jackson.Jackson; @@ -41,19 +41,8 @@ public class SerializationTestUtil { /** * These don't seem to behave well in combination with recursiveComparison. */ - public static final Class[] - TYPES_TO_IGNORE = - new Class[]{ - AtomicInteger.class, - Double.class, - SoftReference.class, - ThreadLocal.class, - User.ShiroUserAdapter.class, - Validator.class, - WeakReference.class, - CompletableFuture.class, - NamespacedStorageProvider.class - }; + public static final Class[] TYPES_TO_IGNORE = new Class[]{AtomicInteger.class, Double.class, SoftReference.class, ThreadLocal.class, User.ShiroUserAdapter.class, Validator.class, WeakReference.class, CompletableFuture.class, NamespacedStorageProvider.class + }; private final JavaType type; private final Validator validator = Validators.newValidator(); @@ -74,7 +63,10 @@ public static SerializationTestUtil forType(Class type) { } public static SerializationTestUtil forArrayType(TypeReference elementType) { - return new SerializationTestUtil<>(Jackson.MAPPER.getTypeFactory().constructArrayType(Jackson.MAPPER.getTypeFactory().constructType(elementType))); + return new SerializationTestUtil<>( + Jackson.MAPPER.getTypeFactory() + .constructArrayType( + Jackson.MAPPER.getTypeFactory().constructType(elementType))); } public SerializationTestUtil objectMappers(ObjectMapper... objectMappers) { @@ -109,11 +101,11 @@ public void test(T value, T expected) throws JSONException, IOException { for (ObjectMapper objectMapper : objectMappers) { try { test( - value, - expected, - objectMapper + value, + expected, + objectMapper ); - } catch (Exception|Error e) { + } catch (Exception | Error e) { Class activeView = objectMapper.getSerializationConfig().getActiveView(); throw new IllegalStateException("Serdes failed with object mapper using view '" + activeView + "'", e); } @@ -144,17 +136,18 @@ private void test(T value, T expected, ObjectMapper mapper) throws IOException { // Preliminary check that ids of identifiables are equal if (value instanceof Identifiable identifiableValue) { - assertThat(((Identifiable) copy).getId()) - .as("the serialized value") - .isEqualTo(identifiableValue.getId()); + assertThat(((Identifiable) copy).getId()).as("the serialized value") + .isEqualTo( + identifiableValue.getId()); } - RecursiveComparisonAssert ass = assertThat(copy) - .as("Unequal after copy.") - .usingRecursiveComparison() - .usingOverriddenEquals() - .ignoringFieldsOfTypes(TYPES_TO_IGNORE) - .ignoringFields("metaStorage", "namespacedStorageProvider"); + RecursiveComparisonAssert ass = assertThat(copy).as( + "Unequal after copy.") + .usingRecursiveComparison() + .usingOverriddenEquals() + .ignoringFieldsOfTypes( + TYPES_TO_IGNORE) + .ignoringFields("metaStorage", "namespacedStorageProvider"); // Apply assertion customizations ass = assertCustomizer.apply(ass); diff --git a/backend/src/test/java/com/bakdata/conquery/io/mina/MinaStackTest.java b/backend/src/test/java/com/bakdata/conquery/io/mina/MinaStackTest.java index f5f586a2e6c..3fc88c794ce 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/mina/MinaStackTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/mina/MinaStackTest.java @@ -66,32 +66,32 @@ public static void beforeAll() throws IOException { // Server SERVER = CLUSTER_CONFIG.getClusterAcceptor(OM, new IoHandlerAdapter() { - @Override - public void sessionOpened(IoSession session) { - log.info("Session to {} established", session.getRemoteAddress()); - } - - @Override - public void messageReceived(IoSession session, Object message) { - SERVER_RECEIVED_MESSAGES.add((NetworkMessage) message); - log.trace("Received {} messages", SERVER_RECEIVED_MESSAGES.size()); - } - - @Override - public void exceptionCaught(IoSession session, Throwable cause) { - fail("Server caught an Exception", cause); - } - }, "Server" + @Override + public void sessionOpened(IoSession session) { + log.info("Session to {} established", session.getRemoteAddress()); + } + + @Override + public void messageReceived(IoSession session, Object message) { + SERVER_RECEIVED_MESSAGES.add((NetworkMessage) message); + log.trace("Received {} messages", SERVER_RECEIVED_MESSAGES.size()); + } + + @Override + public void exceptionCaught(IoSession session, Throwable cause) { + fail("Server caught an Exception", cause); + } + }, "Server" ); } private static Stream dataSizes() { return Stream.of( - Arguments.of(DataSize.bytes(10), true), - Arguments.of(DataSize.kibibytes(10), true), - Arguments.of(DataSize.mebibytes(9), true), - Arguments.of(DataSize.mebibytes(10), true) + Arguments.of(DataSize.bytes(10), true), + Arguments.of(DataSize.kibibytes(10), true), + Arguments.of(DataSize.mebibytes(9), true), + Arguments.of(DataSize.mebibytes(10), true) ); } @@ -109,14 +109,14 @@ public void beforeEach() { void smokeTest() { final NioSocketConnector client = CLUSTER_CONFIG.getClusterConnector( - OM, - new IoHandlerAdapter() { - @Override - public void sessionOpened(IoSession session) { - log.info("Session to {} established", session.getRemoteAddress()); - } - }, - "Client" + OM, + new IoHandlerAdapter() { + @Override + public void sessionOpened(IoSession session) { + log.info("Session to {} established", session.getRemoteAddress()); + } + }, + "Client" ); try { @@ -136,8 +136,7 @@ public void sessionOpened(IoSession session) { assertThat(SERVER_RECEIVED_MESSAGES).containsExactlyInAnyOrder(input); clientSession.closeNow().awaitUninterruptibly(); - } - finally { + } finally { client.dispose(); } @@ -161,51 +160,54 @@ void concurrentWriting() { for (int clientI = 0; clientI < clientCount; clientI++) { final int clientNumber = clientI; final CompletableFuture clientThread = CompletableFuture.runAsync( - () -> { - final NioSocketConnector client = CLUSTER_CONFIG.getClusterConnector( - OM, - new IoHandlerAdapter() { - @Override - public void sessionOpened(IoSession session) { - log.info("Session to {} established", session.getRemoteAddress()); - } - - @Override - public void messageSent(IoSession session, Object message) { - log.trace("Message written: {} bytes", ((TestMessage) message).data.getBytes().length); - } - - @Override - public void exceptionCaught(IoSession session, Throwable cause) { - fail("Client[%d] caught an Exception".formatted(clientNumber), cause); - } - }, - "Client" - ); - try { - // Connect - final ConnectFuture connect = client.connect(SERVER.getLocalAddress()); - connect.awaitUninterruptibly(); - final IoSession clientSession = connect.getSession(); - - for (int i = 0; i < messagesPerClient; i++) { - final NetworkMessage input = - new TestMessage("concurrentWrite_" + clientNumber, RandomStringUtils.randomAscii(minMessageLength, maxMessageLength)); - - final WriteFuture writeFuture = clientSession.write(input); - writeFuture.addListener((f) -> { - if (!((WriteFuture) f).isWritten()) { - fail("Failed to write a message"); - } - messagesWritten.add(input); - }); - writeFuture.awaitUninterruptibly(); + () -> { + final NioSocketConnector client = CLUSTER_CONFIG.getClusterConnector( + OM, + new IoHandlerAdapter() { + @Override + public void sessionOpened(IoSession session) { + log.info("Session to {} established", session.getRemoteAddress()); } + + @Override + public void messageSent(IoSession session, Object message) { + log.trace( + "Message written: {} bytes", + ((TestMessage) message).data.getBytes().length); + } + + @Override + public void exceptionCaught(IoSession session, Throwable cause) { + fail("Client[%d] caught an Exception".formatted(clientNumber), cause); + } + }, + "Client" + ); + try { + // Connect + final ConnectFuture connect = client.connect(SERVER.getLocalAddress()); + connect.awaitUninterruptibly(); + final IoSession clientSession = connect.getSession(); + + for (int i = 0; i < messagesPerClient; i++) { + final NetworkMessage input = new TestMessage( + "concurrentWrite_" + clientNumber, + RandomStringUtils.randomAscii(minMessageLength, maxMessageLength)); + + final WriteFuture writeFuture = clientSession.write(input); + writeFuture.addListener((f) -> { + if (!((WriteFuture) f).isWritten()) { + fail("Failed to write a message"); + } + messagesWritten.add(input); + }); + writeFuture.awaitUninterruptibly(); } - finally { - client.dispose(); - } - }, executorService + } finally { + client.dispose(); + } + }, + executorService ); clientThreads.add(clientThread); } @@ -217,14 +219,17 @@ public void exceptionCaught(IoSession session, Throwable cause) { // Wait until all messages are received await().atMost(10, TimeUnit.SECONDS) - .alias(String.format("Send and received same amount of messages (%s < %s)", SERVER_RECEIVED_MESSAGES.size(), messagesWritten.size())) - .until(() -> SERVER_RECEIVED_MESSAGES.size() >= messagesWritten.size()); + .alias( + String.format( + "Send and received same amount of messages (%s < %s)", + SERVER_RECEIVED_MESSAGES.size(), + messagesWritten.size())) + .until(() -> SERVER_RECEIVED_MESSAGES.size() >= messagesWritten.size()); // Check that the messages are correct assertThat(SERVER_RECEIVED_MESSAGES).containsExactlyInAnyOrderElementsOf(messagesWritten); - } - finally { + } finally { executorService.shutdownNow(); } @@ -234,18 +239,19 @@ public void exceptionCaught(IoSession session, Throwable cause) { @MethodSource("dataSizes") void messageSizes(DataSize dataSize) { final NioSocketConnector client = CLUSTER_CONFIG.getClusterConnector( - OM, - new IoHandlerAdapter() { - @Override - public void sessionOpened(IoSession session) { - log.info("Session to {} established", session.getRemoteAddress()); - } - - @Override - public void exceptionCaught(IoSession session, Throwable cause) { - log.trace("Failed to write message (probably expected)", cause); - } - }, "Client" + OM, + new IoHandlerAdapter() { + @Override + public void sessionOpened(IoSession session) { + log.info("Session to {} established", session.getRemoteAddress()); + } + + @Override + public void exceptionCaught(IoSession session, Throwable cause) { + log.trace("Failed to write message (probably expected)", cause); + } + }, + "Client" ); try { @@ -255,24 +261,25 @@ public void exceptionCaught(IoSession session, Throwable cause) { connect.awaitUninterruptibly(); final IoSession clientSession = connect.getSession(); - final NetworkMessage input = new TestMessage("sizes_" + dataSize, RandomStringUtils.randomAscii(toIntExact(dataSize.toBytes()))); + final NetworkMessage input = new TestMessage( + "sizes_" + dataSize, + RandomStringUtils.randomAscii(toIntExact(dataSize.toBytes()))); final WriteFuture write = clientSession.write(input); write.awaitUninterruptibly(); - assertThat(write.isWritten()) - .describedAs(() -> write.getException().toString()) - .isTrue(); + assertThat(write.isWritten()).describedAs(() -> write.getException().toString()).isTrue(); clientSession.closeNow().awaitUninterruptibly(); // Wait until all messages are received await().atMost(10, TimeUnit.SECONDS) - .alias(String.format("Receive the message (%s)", SERVER_RECEIVED_MESSAGES.size())) - .until(() -> SERVER_RECEIVED_MESSAGES.size() == 1); - } - finally { + .alias( + String.format("Receive the message (%s)", SERVER_RECEIVED_MESSAGES.size())) + .until( + () -> SERVER_RECEIVED_MESSAGES.size() == 1); + } finally { client.dispose(); } } @@ -281,14 +288,14 @@ public void exceptionCaught(IoSession session, Throwable cause) { void differentMessageTypes() { final NioSocketConnector client = CLUSTER_CONFIG.getClusterConnector( - OM, - new IoHandlerAdapter() { - @Override - public void sessionOpened(IoSession session) { - log.info("Session to {} established", session.getRemoteAddress()); - } - }, - "Client" + OM, + new IoHandlerAdapter() { + @Override + public void sessionOpened(IoSession session) { + log.info("Session to {} established", session.getRemoteAddress()); + } + }, + "Client" ); try { @@ -302,20 +309,19 @@ public void sessionOpened(IoSession session) { final NetworkMessage input1 = new TestMessage("messageTypes", RandomStringUtils.randomAscii(1000)); final WriteFuture write1 = session.write(input1); - final NetworkMessage input2 = ForwardToWorker.create(new WorkerId(new DatasetId("dataset"), "worker"), new RequestConsistency()); + final NetworkMessage input2 = ForwardToWorker.create( + new WorkerId(new DatasetId("dataset"), "worker"), + new RequestConsistency()); final WriteFuture write2 = session.write(input2); write1.awaitUninterruptibly(); write2.awaitUninterruptibly(); await().atMost(1, TimeUnit.MINUTES).until(() -> !SERVER_RECEIVED_MESSAGES.isEmpty()); - assertThat(SERVER_RECEIVED_MESSAGES) - .containsExactlyInAnyOrder(input1, input2) - .usingRecursiveComparison(); + assertThat(SERVER_RECEIVED_MESSAGES).containsExactlyInAnyOrder(input1, input2).usingRecursiveComparison(); session.closeNow().awaitUninterruptibly(); - } - finally { + } finally { client.dispose(); } } @@ -337,23 +343,24 @@ void edgeBufferingTest() { final ConcurrentLinkedQueue> messagesWritten = new ConcurrentLinkedQueue<>(); final NioSocketConnector client = CLUSTER_CONFIG.getClusterConnector( - OM, - new IoHandlerAdapter() { - @Override - public void sessionOpened(IoSession session) { - log.info("Session to {} established", session.getRemoteAddress()); - } - - @Override - public void messageSent(IoSession session, Object message) { - log.trace("Message written: {} bytes", ((TestMessage) message).data.getBytes().length); - } - - @Override - public void exceptionCaught(IoSession session, Throwable cause) { - fail("Client caught an Exception", cause); - } - }, "Client" + OM, + new IoHandlerAdapter() { + @Override + public void sessionOpened(IoSession session) { + log.info("Session to {} established", session.getRemoteAddress()); + } + + @Override + public void messageSent(IoSession session, Object message) { + log.trace("Message written: {} bytes", ((TestMessage) message).data.getBytes().length); + } + + @Override + public void exceptionCaught(IoSession session, Throwable cause) { + fail("Client caught an Exception", cause); + } + }, + "Client" ); try { @@ -363,7 +370,9 @@ public void exceptionCaught(IoSession session, Throwable cause) { final IoSession clientSession = connect.getSession(); for (int i = 0; i < totalMessages; i++) { - final NetworkMessage input = new TestMessage("spillBufferTest", RandomStringUtils.randomAscii(minMessageLength, maxMessageLength)); + final NetworkMessage input = new TestMessage( + "spillBufferTest", + RandomStringUtils.randomAscii(minMessageLength, maxMessageLength)); final WriteFuture writeFuture = clientSession.write(input); writeFuture.addListener((f) -> { @@ -378,14 +387,17 @@ public void exceptionCaught(IoSession session, Throwable cause) { log.info("Waiting to receive all sent messages"); // Wait until all messages are received await().atMost(10, TimeUnit.SECONDS) - .alias(String.format("Send and received same amount of messages (%s < %s)", SERVER_RECEIVED_MESSAGES.size(), messagesWritten.size())) - .until(() -> SERVER_RECEIVED_MESSAGES.size() >= messagesWritten.size()); + .alias( + String.format( + "Send and received same amount of messages (%s < %s)", + SERVER_RECEIVED_MESSAGES.size(), + messagesWritten.size())) + .until(() -> SERVER_RECEIVED_MESSAGES.size() >= messagesWritten.size()); // Check that the messages are correct assertThat(SERVER_RECEIVED_MESSAGES).containsExactlyInAnyOrderElementsOf(messagesWritten); - } - finally { + } finally { client.dispose(); } diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java index b3a62ec159c..5863ff96efe 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java @@ -62,19 +62,17 @@ public class ResultTestUtil { CONCEPT.setNamespacedStorageProvider(STORAGE); try { CONCEPT.init(); - } - catch (Exception e) { + } catch (Exception e) { throw new RuntimeException(e); } } public static List getIdFields() { - return Stream.of("id1", "id2") - .map(name -> { - ExternalResultInfo info = new ExternalResultInfo(name, ResultType.Primitive.STRING); - info.addSemantics(new SemanticType.IdT("ID")); - return info; - }).collect(Collectors.toList()); + return Stream.of("id1", "id2").map(name -> { + ExternalResultInfo info = new ExternalResultInfo(name, ResultType.Primitive.STRING); + info.addSemantics(new SemanticType.IdT("ID")); + return info; + }).collect(Collectors.toList()); } @NotNull @@ -83,9 +81,11 @@ public static ManagedQuery getTestQuery() { @Override public List collectResultInfos() { return getResultTypes().stream() - .map(TypedSelectDummy::new) - .map(select -> new SelectResultInfo(select, new CQConcept(), Collections.emptySet())) - .collect(Collectors.toList()); + .map(TypedSelectDummy::new) + .map( + select -> new SelectResultInfo(select, new CQConcept(), Collections.emptySet())) + .collect( + Collectors.toList()); } @Override @@ -97,56 +97,51 @@ public Stream streamResults(OptionalLong maybeLimit) { @NotNull public static List getResultTypes() { - return List.of(ResultType.Primitive.BOOLEAN, - ResultType.Primitive.INTEGER, - ResultType.Primitive.NUMERIC, - ResultType.Primitive.DATE, - ResultType.Primitive.DATE_RANGE, - ResultType.Primitive.STRING, - ResultType.Primitive.MONEY, - new ResultType.ListT(ResultType.Primitive.BOOLEAN), - new ResultType.ListT(ResultType.Primitive.DATE_RANGE), - new ResultType.ListT(ResultType.Primitive.STRING) + return List.of( + ResultType.Primitive.BOOLEAN, + ResultType.Primitive.INTEGER, + ResultType.Primitive.NUMERIC, + ResultType.Primitive.DATE, + ResultType.Primitive.DATE_RANGE, + ResultType.Primitive.STRING, + ResultType.Primitive.MONEY, + new ResultType.ListT(ResultType.Primitive.BOOLEAN), + new ResultType.ListT(ResultType.Primitive.DATE_RANGE), + new ResultType.ListT(ResultType.Primitive.STRING) ); } @NotNull public static List getTestEntityResults() { - return List.of(new SinglelineEntityResult("1", new Object[]{ - Boolean.TRUE, - 2345634, - 123423.34, - 5646, - List.of(345, 534), - "test_string", - new BigDecimal("45.21"), - List.of(true, false), - List.of(List.of(345, 534), List.of(1, 2)), - List.of("fizz", "buzz") - } - ), - new SinglelineEntityResult("2", new Object[]{ - Boolean.FALSE, null, null, null, null, null, null, List.of(), List.of(List.of(1234, Integer.MAX_VALUE)), List.of() - } - ), - new SinglelineEntityResult("2", new Object[]{Boolean.TRUE, null, null, null, null, null, null, List.of(false, false), null, null}), - new MultilineEntityResult("3", - List.of(new Object[]{Boolean.FALSE, null, null, null, null, null, null, List.of(false), null, null}, - new Object[]{Boolean.TRUE, null, null, null, null, null, null, null, null, null}, - new Object[]{ - Boolean.TRUE, - null, - null, - null, - null, - null, - new BigDecimal("4.00"), - List.of(true, false, true, false), - null, - null - } - ) - ) + return List.of( + new SinglelineEntityResult( + "1", + new Object[]{Boolean.TRUE, 2345634, 123423.34, 5646, List.of(345, 534), "test_string", new BigDecimal( + "45.21"), List.of(true, false), List.of(List.of(345, 534), List.of(1, 2)), List.of("fizz", "buzz") + } + ), + new SinglelineEntityResult( + "2", + new Object[]{Boolean.FALSE, null, null, null, null, null, null, List.of(), List.of( + List.of(1234, Integer.MAX_VALUE)), List.of() + } + ), + new SinglelineEntityResult( + "2", + new Object[]{Boolean.TRUE, null, null, null, null, null, null, List.of(false, false), null, null}), + new MultilineEntityResult( + "3", + List.of( + new Object[]{Boolean.FALSE, null, null, null, null, null, null, List.of(false), null, null}, + new Object[]{Boolean.TRUE, null, null, null, null, null, null, null, null, null}, + new Object[]{Boolean.TRUE, null, null, null, null, null, new BigDecimal("4.00"), List.of( + true, + false, + true, + false), null, null + } + ) + ) ); } diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java index 31217c2d2f8..d9014ecb787 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java @@ -55,24 +55,32 @@ public class ArrowResultGenerationTest { public static final ConqueryConfig CONFIG = new ConqueryConfig(); private static final int BATCH_SIZE = 2; - private static final PrintSettings PRINT_SETTINGS = new PrintSettings(false, Locale.ROOT, CONFIG, null, (selectInfo) -> selectInfo.getSelect().getLabel()); + private static final PrintSettings PRINT_SETTINGS = new PrintSettings( + false, + Locale.ROOT, + CONFIG, + null, + (selectInfo) -> selectInfo.getSelect().getLabel()); public static String readTSV(InputStream inputStream) throws IOException { StringJoiner stringJoiner = new StringJoiner("\n"); try (ArrowStreamReader arrowReader = new ArrowStreamReader(inputStream, ROOT_ALLOCATOR)) { log.info("Reading the produced arrow data."); VectorSchemaRoot readRoot = arrowReader.getVectorSchemaRoot(); - stringJoiner.add(readRoot.getSchema().getFields().stream().map(Field::getName).collect(Collectors.joining("\t"))); + stringJoiner.add( + readRoot.getSchema().getFields().stream().map(Field::getName).collect(Collectors.joining("\t"))); readRoot.setRowCount(BATCH_SIZE); while (arrowReader.loadNextBatch()) { List vectors = readRoot.getFieldVectors(); for (int rowI = 0; rowI < readRoot.getRowCount(); rowI++) { final int currentRow = rowI; - stringJoiner.add(vectors.stream() - .map(vec -> vec.getObject(currentRow)) - .map(ArrowResultGenerationTest::getPrintValue) - .collect(Collectors.joining("\t"))); + stringJoiner.add( + vectors.stream() + .map(vec -> vec.getObject(currentRow)) + .map( + ArrowResultGenerationTest::getPrintValue) + .collect(Collectors.joining("\t"))); } } } @@ -80,38 +88,33 @@ public static String readTSV(InputStream inputStream) throws IOException { } public static String generateExpectedTSV(List results, List resultInfos) { - String expected = - results.stream() - .map(res -> { - StringJoiner lineJoiner = new StringJoiner("\n"); + String expected = results.stream().map(res -> { + StringJoiner lineJoiner = new StringJoiner("\n"); - for (Object[] line : res.listResultLines()) { - StringJoiner valueJoiner = new StringJoiner("\t"); + for (Object[] line : res.listResultLines()) { + StringJoiner valueJoiner = new StringJoiner("\t"); - valueJoiner.add(String.valueOf(res.getEntityId())); - valueJoiner.add(String.valueOf(res.getEntityId())); + valueJoiner.add(String.valueOf(res.getEntityId())); + valueJoiner.add(String.valueOf(res.getEntityId())); - for (int lIdx = 0; lIdx < line.length; lIdx++) { - Object val = line[lIdx]; - ResultInfo info = resultInfos.get(lIdx); + for (int lIdx = 0; lIdx < line.length; lIdx++) { + Object val = line[lIdx]; + ResultInfo info = resultInfos.get(lIdx); - valueJoiner.add(getPrintValue(val, info.getType())); - } + valueJoiner.add(getPrintValue(val, info.getType())); + } - lineJoiner.add(valueJoiner.toString()); - } - return lineJoiner.toString(); - }).collect(Collectors.joining("\n")); + lineJoiner.add(valueJoiner.toString()); + } + return lineJoiner.toString(); + }).collect(Collectors.joining("\n")); return Stream.concat( - // Id column headers - getIdFields().stream().map(i -> i.defaultColumnName(PRINT_SETTINGS)), - // result column headers - getResultTypes().stream().map(ResultType::typeInfo) - ) - .collect(Collectors.joining("\t")) - + "\n" - + expected; + // Id column headers + getIdFields().stream().map(i -> i.defaultColumnName(PRINT_SETTINGS)), + // result column headers + getResultTypes().stream().map(ResultType::typeInfo) + ).collect(Collectors.joining("\t")) + "\n" + expected; } private static String getPrintValue(Object obj, ResultType type) { @@ -142,7 +145,7 @@ private static String getPrintValue(Object obj, ResultType type) { return sb.toString(); } if (obj instanceof Collection col) { - // Workaround: Arrow deserializes lists as a JsonStringArrayList which has a JSON String method + // Workaround: Arrow deserializes lists as a JsonStringArrayList which has a JSON String method ResultType elemType = ((ResultType.ListT) type).getElementType(); return col.stream().map(v -> getPrintValue(v, elemType)).collect(Collectors.joining(",", "[", "]")); } @@ -152,7 +155,10 @@ private static String getPrintValue(Object obj, ResultType type) { private static String getPrintValue(Object obj) { if (obj instanceof JsonStringArrayList) { // Workaround: Arrow deserializes lists as a JsonStringArrayList which has a JSON String method - return new ArrayList<>((JsonStringArrayList) obj).stream().map(ArrowResultGenerationTest::getPrintValue).collect(Collectors.joining(",", "[", "]")); + return new ArrayList<>((JsonStringArrayList) obj).stream() + .map( + ArrowResultGenerationTest::getPrintValue) + .collect(Collectors.joining(",", "[", "]")); } return Objects.toString(obj); } @@ -164,9 +170,10 @@ void generateFieldsIdMapping() { List fields = generateFields(getIdFields(), uniqueNamer, PRINT_SETTINGS); assertThat(fields).containsExactlyElementsOf( - List.of(new Field("id1", FieldType.nullable(new ArrowType.Utf8()), null), - new Field("id2", FieldType.nullable(new ArrowType.Utf8()), null) - )); + List.of( + new Field("id1", FieldType.nullable(new ArrowType.Utf8()), null), + new Field("id2", FieldType.nullable(new ArrowType.Utf8()), null) + )); } @@ -176,46 +183,62 @@ void generateFieldsValue() { List resultInfos = getResultTypes().stream() - .map(TypedSelectDummy::new) - .map(select -> new SelectResultInfo(select, new CQConcept(), Collections.emptySet())) - .collect(Collectors.toList()); - - List fields = generateFields(resultInfos, - // Custom column namer so we don't require a dataset registry - uniqueNamer, PRINT_SETTINGS + .map(TypedSelectDummy::new) + .map( + select -> new SelectResultInfo(select, new CQConcept(), Collections.emptySet())) + .collect( + Collectors.toList()); + + List fields = generateFields( + resultInfos, + // Custom column namer so we don't require a dataset registry + uniqueNamer, + PRINT_SETTINGS ); assertThat(fields).containsExactlyElementsOf( - List.of(new Field("BOOLEAN", FieldType.nullable(ArrowType.Bool.INSTANCE), null), - new Field("INTEGER", FieldType.nullable(new ArrowType.Int(32, true)), null), - new Field("NUMERIC", FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), null), - new Field("DATE", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null), - new Field("DATE_RANGE", - FieldType.nullable(ArrowType.Struct.INSTANCE), - List.of(new Field("min", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null), - new Field("max", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null) - ) - ), - new Field("STRING", FieldType.nullable(new ArrowType.Utf8()), null), - new Field("MONEY", FieldType.nullable(new ArrowType.Int(32, true)), null), - new Field("LIST[BOOLEAN]", - FieldType.nullable(ArrowType.List.INSTANCE), - List.of(new Field("LIST[BOOLEAN]", FieldType.nullable(ArrowType.Bool.INSTANCE), null)) - ), - new Field("LIST[DATE_RANGE]", - FieldType.nullable(ArrowType.List.INSTANCE), - List.of(new Field("LIST[DATE_RANGE]", - FieldType.nullable(ArrowType.Struct.INSTANCE), - List.of(new Field("min", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null), - new Field("max", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null) - ) - )) - ), - new Field("LIST[STRING]", - FieldType.nullable(ArrowType.List.INSTANCE), - List.of(new Field("LIST[STRING]", FieldType.nullable(new ArrowType.Utf8()), null)) - ) - )); + List.of( + new Field("BOOLEAN", FieldType.nullable(ArrowType.Bool.INSTANCE), null), + new Field("INTEGER", FieldType.nullable(new ArrowType.Int(32, true)), null), + new Field( + "NUMERIC", + FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + null), + new Field("DATE", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null), + new Field( + "DATE_RANGE", + FieldType.nullable(ArrowType.Struct.INSTANCE), + List.of( + new Field("min", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null), + new Field("max", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null) + ) + ), + new Field("STRING", FieldType.nullable(new ArrowType.Utf8()), null), + new Field("MONEY", FieldType.nullable(new ArrowType.Int(32, true)), null), + new Field( + "LIST[BOOLEAN]", + FieldType.nullable(ArrowType.List.INSTANCE), + List.of(new Field("LIST[BOOLEAN]", FieldType.nullable(ArrowType.Bool.INSTANCE), null)) + ), + new Field( + "LIST[DATE_RANGE]", + FieldType.nullable(ArrowType.List.INSTANCE), + List.of( + new Field( + "LIST[DATE_RANGE]", + FieldType.nullable(ArrowType.Struct.INSTANCE), + List.of( + new Field("min", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null), + new Field("max", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null) + ) + )) + ), + new Field( + "LIST[STRING]", + FieldType.nullable(ArrowType.List.INSTANCE), + List.of(new Field("LIST[STRING]", FieldType.nullable(new ArrowType.Utf8()), null)) + ) + )); } @@ -226,11 +249,12 @@ void writeAndRead() throws IOException { I18n.init(); // Prepare every input data - PrintSettings printSettings = new PrintSettings(false, - Locale.ROOT, - CONFIG, - (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), - (selectInfo) -> selectInfo.getSelect().getLabel() + PrintSettings printSettings = new PrintSettings( + false, + Locale.ROOT, + CONFIG, + (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), + (selectInfo) -> selectInfo.getSelect().getLabel() ); // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized List results = getTestEntityResults(); @@ -240,13 +264,14 @@ void writeAndRead() throws IOException { // First we write to the buffer, than we read from it and parse it as TSV ByteArrayOutputStream output = new ByteArrayOutputStream(); - renderToStream((root) -> new ArrowStreamWriter(root, new DictionaryProvider.MapDictionaryProvider(), output), - printSettings, - new ArrowConfig(BATCH_SIZE), - getIdFields(), - mquery.collectResultInfos(), - mquery.streamResults(OptionalLong.empty()), - new ArrowResultPrinters() + renderToStream( + (root) -> new ArrowStreamWriter(root, new DictionaryProvider.MapDictionaryProvider(), output), + printSettings, + new ArrowConfig(BATCH_SIZE), + getIdFields(), + mquery.collectResultInfos(), + mquery.streamResults(OptionalLong.empty()), + new ArrowResultPrinters() ); InputStream inputStream = new ByteArrayInputStream(output.toByteArray()); diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java index 7cd4a3f17ab..52ce6aa4349 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java @@ -34,10 +34,12 @@ @Slf4j public class CsvResultGenerationTest { - public static final ConqueryConfig CONFIG = new ConqueryConfig() {{ - // Suppress java.lang.NoClassDefFoundError: com/bakdata/conquery/io/jackson/serializer/CurrencyUnitDeserializer - setStorage(new NonPersistentStoreFactory()); - }}; + public static final ConqueryConfig CONFIG = new ConqueryConfig() { + { + // Suppress java.lang.NoClassDefFoundError: com/bakdata/conquery/io/jackson/serializer/CurrencyUnitDeserializer + setStorage(new NonPersistentStoreFactory()); + } + }; static { I18n.init(); @@ -49,11 +51,12 @@ void writeAndRead(String charsetName) throws IOException { final Charset charset = Charset.forName(charsetName); // Prepare every input data - final PrintSettings printSettings = new PrintSettings(true, - Locale.GERMANY, - CONFIG, - (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), - (selectInfo) -> selectInfo.getSelect().getLabel() + final PrintSettings printSettings = new PrintSettings( + true, + Locale.GERMANY, + CONFIG, + (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), + (selectInfo) -> selectInfo.getSelect().getLabel() ); // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized final List results = getTestEntityResults(); @@ -66,7 +69,12 @@ void writeAndRead(String charsetName) throws IOException { try (Writer writer = new BufferedWriter(new OutputStreamWriter(bufferOut, charset))) { final CsvRenderer renderer = new CsvRenderer(CONFIG.getCsv().createWriter(writer), printSettings); - renderer.toCSV(getIdFields(), mquery.collectResultInfos(), mquery.streamResults(OptionalLong.empty()), printSettings, charset); + renderer.toCSV( + getIdFields(), + mquery.collectResultInfos(), + mquery.streamResults(OptionalLong.empty()), + printSettings, + charset); } final String computed = bufferOut.toString(charset); @@ -81,43 +89,48 @@ void writeAndRead(String charsetName) throws IOException { } - private String generateExpectedCSV(List results, List resultInfos, PrintSettings printSettings, PrinterFactory printerFactory) { + private String generateExpectedCSV( + List results, + List resultInfos, + PrintSettings printSettings, + PrinterFactory printerFactory) { final List expected = new ArrayList<>(); - expected.add(getIdFields().stream().map(info -> info.defaultColumnName(printSettings)).collect(Collectors.joining(",")) - + "," - + getResultTypes().stream() - .map(ResultType::typeInfo) - .collect(Collectors.joining(",")) - + "\n"); + expected.add( + getIdFields().stream() + .map(info -> info.defaultColumnName(printSettings)) + .collect( + Collectors.joining(",")) + "," + getResultTypes().stream() + .map(ResultType::typeInfo) + .collect( + Collectors.joining(",")) + "\n"); final String delimiter = String.valueOf(CONFIG.getCsv().getDelimeter()); - results - .forEach(res -> { + results.forEach(res -> { - for (Object[] line : res.listResultLines()) { - final StringJoiner valueJoiner = new StringJoiner(","); + for (Object[] line : res.listResultLines()) { + final StringJoiner valueJoiner = new StringJoiner(","); - valueJoiner.add(String.valueOf(res.getEntityId())); - valueJoiner.add(String.valueOf(res.getEntityId())); + valueJoiner.add(String.valueOf(res.getEntityId())); + valueJoiner.add(String.valueOf(res.getEntityId())); - for (int lIdx = 0; lIdx < line.length; lIdx++) { - final Object val = line[lIdx]; + for (int lIdx = 0; lIdx < line.length; lIdx++) { + final Object val = line[lIdx]; - if (val == null) { - valueJoiner.add(""); - continue; - } + if (val == null) { + valueJoiner.add(""); + continue; + } - final ResultInfo info = resultInfos.get(lIdx); - final String printVal = (String) info.createPrinter(printerFactory, printSettings).apply(val); + final ResultInfo info = resultInfos.get(lIdx); + final String printVal = (String) info.createPrinter(printerFactory, printSettings).apply(val); - valueJoiner.add(printVal.contains(delimiter) ? "\"" + printVal + "\"" : printVal); - } + valueJoiner.add(printVal.contains(delimiter) ? "\"" + printVal + "\"" : printVal); + } - expected.add(valueJoiner + "\n"); - } - }); + expected.add(valueJoiner + "\n"); + } + }); return String.join("", expected); } diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java index bb72707b75b..d68bddc0e11 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java @@ -52,10 +52,12 @@ @Slf4j public class ExcelResultRenderTest { - public static final ConqueryConfig CONFIG = new ConqueryConfig() {{ - // Suppress java.lang.NoClassDefFoundError: com/bakdata/conquery/io/jackson/serializer/CurrencyUnitDeserializer - setStorage(new NonPersistentStoreFactory()); - }}; + public static final ConqueryConfig CONFIG = new ConqueryConfig() { + { + // Suppress java.lang.NoClassDefFoundError: com/bakdata/conquery/io/jackson/serializer/CurrencyUnitDeserializer + setStorage(new NonPersistentStoreFactory()); + } + }; private static final List printIdFields = List.of("id1", "id2"); static { @@ -65,11 +67,12 @@ public class ExcelResultRenderTest { @Test void writeAndRead() throws IOException { // Prepare every input data - final PrintSettings printSettings = new PrintSettings(true, - Locale.GERMAN, - CONFIG, - (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), - (selectInfo) -> selectInfo.getSelect().getLabel() + final PrintSettings printSettings = new PrintSettings( + true, + Locale.GERMAN, + CONFIG, + (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), + (selectInfo) -> selectInfo.getSelect().getLabel() ); // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized final List results = getTestEntityResults(); @@ -84,7 +87,13 @@ void writeAndRead() throws IOException { final ExcelRenderer renderer = new ExcelRenderer(new ExcelConfig(), printSettings); - renderer.renderToStream(ResultTestUtil.getIdFields(), mquery, output, OptionalLong.empty(), printSettings, metaStorage); + renderer.renderToStream( + ResultTestUtil.getIdFields(), + mquery, + output, + OptionalLong.empty(), + printSettings, + metaStorage); final InputStream inputStream = new ByteArrayInputStream(output.toByteArray()); @@ -92,17 +101,19 @@ void writeAndRead() throws IOException { final List computed = readComputed(inputStream, printSettings); // We have to do some magic here to emulate the excel printed results. - PrintSettings tsvPrintSettings = new PrintSettings(true, - Locale.GERMAN, - CONFIG, - (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), - (selectInfo) -> selectInfo.getSelect().getLabel() + PrintSettings tsvPrintSettings = new PrintSettings( + true, + Locale.GERMAN, + CONFIG, + (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), + (selectInfo) -> selectInfo.getSelect().getLabel() ); - final List expected = generateExpectedTSV(results, - mquery.getResultInfos(), - tsvPrintSettings, - new StringResultPrinters(ExcelResultPrinters.NEGATIVE_INF, ExcelResultPrinters.POSITIVE_INF) + final List expected = generateExpectedTSV( + results, + mquery.getResultInfos(), + tsvPrintSettings, + new StringResultPrinters(ExcelResultPrinters.NEGATIVE_INF, ExcelResultPrinters.POSITIVE_INF) ); log.info("Wrote and than read this excel data: {}", computed); @@ -116,7 +127,13 @@ void writeAndRead() throws IOException { User user = new User("test", "test", metaStorage); user.updateStorage(); - return new ManagedQuery(mock(Query.class), user.getId(), new DatasetId(ExcelResultRenderTest.class.getSimpleName()), metaStorage, null, CONFIG) { + return new ManagedQuery( + mock(Query.class), + user.getId(), + new DatasetId(ExcelResultRenderTest.class.getSimpleName()), + metaStorage, + null, + CONFIG) { @Override public Stream streamResults(OptionalLong maybeLimit) { return results.stream(); @@ -125,9 +142,11 @@ public Stream streamResults(OptionalLong maybeLimit) { @Override public List getResultInfos() { return getResultTypes().stream() - .map(ResultTestUtil.TypedSelectDummy::new) - .map(select -> new SelectResultInfo(select, new CQConcept(), Collections.emptySet())) - .collect(Collectors.toList()); + .map(ResultTestUtil.TypedSelectDummy::new) + .map( + select -> new SelectResultInfo(select, new CQConcept(), Collections.emptySet())) + .collect( + Collectors.toList()); } }; } @@ -159,9 +178,16 @@ private List readComputed(InputStream inputStream, PrintSettings setting private List generateExpectedTSV( - List results, List resultInfos, PrintSettings printSettings, PrinterFactory printerFactory) { + List results, + List resultInfos, + PrintSettings printSettings, + PrinterFactory printerFactory) { final List expected = new ArrayList<>(); - expected.add(String.join("\t", printIdFields) + "\t" + getResultTypes().stream().map(ResultType::typeInfo).collect(Collectors.joining("\t"))); + expected.add( + String.join("\t", printIdFields) + "\t" + getResultTypes().stream() + .map(ResultType::typeInfo) + .collect( + Collectors.joining("\t"))); for (EntityResult res : results) { for (Object[] line : res.listResultLines()) { diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java index 9d7e064a366..b956ec2b934 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/parquet/ParquetResultGenerationTest.java @@ -45,53 +45,111 @@ public class ParquetResultGenerationTest { public static final ConqueryConfig CONFIG = new ConqueryConfig(); - private static final PrintSettings - PRINT_SETTINGS = - new PrintSettings(false, Locale.ROOT, CONFIG, null, (selectInfo) -> selectInfo.getSelect().getLabel()); + private static final PrintSettings PRINT_SETTINGS = new PrintSettings( + false, + Locale.ROOT, + CONFIG, + null, + (selectInfo) -> selectInfo.getSelect().getLabel()); @Test void generateSchema() { final UniqueNamer uniqueNamer = new UniqueNamer(PRINT_SETTINGS); - List resultInfos = getResultTypes().stream().map(TypedSelectDummy::new) - .map(select -> new SelectResultInfo(select, new CQConcept(), Collections.emptySet())).collect(Collectors.toList()); + List resultInfos = getResultTypes().stream() + .map(TypedSelectDummy::new) + .map( + select -> new SelectResultInfo(select, new CQConcept(), Collections.emptySet())) + .collect( + Collectors.toList()); - final MessageType messageType = EntityResultWriteSupport.generateSchema(getIdFields(), resultInfos, uniqueNamer, PRINT_SETTINGS); + final MessageType messageType = EntityResultWriteSupport.generateSchema( + getIdFields(), + resultInfos, + uniqueNamer, + PRINT_SETTINGS); assertThat(messageType).isEqualTo( - Types.buildMessage() - .optional(BINARY).as(stringType()).named("id1") - .optional(BINARY).as(stringType()).named("id2") - .optional(BOOLEAN).named("BOOLEAN") - .optional(INT32).as(LogicalTypeAnnotation.intType(32, true)).named("INTEGER") - .optional(DOUBLE).named("NUMERIC") - .optional(INT32).as(LogicalTypeAnnotation.dateType()).named("DATE") - .optionalGroup() - .optional(INT32).as(LogicalTypeAnnotation.dateType()).named("min") - .optional(INT32).as(LogicalTypeAnnotation.dateType()).named("max") - .named("DATE_RANGE") - .optional(BINARY).as(stringType()).named("STRING") - .optional(INT32).as(LogicalTypeAnnotation.intType(32, true)).named("MONEY") - .optionalGroup().as(LogicalTypeAnnotation.listType()) - .repeatedGroup() - .optional(BOOLEAN).named("element") - .named("list") - .named("LIST[BOOLEAN]") - .optionalGroup().as(LogicalTypeAnnotation.listType()) - .repeatedGroup() - .optionalGroup() - .optional(INT32).as(LogicalTypeAnnotation.dateType()).named("min") - .optional(INT32).as(LogicalTypeAnnotation.dateType()).named("max") - .named("element") - .named("list") - .named("LIST[DATE_RANGE]") - .optionalGroup().as(LogicalTypeAnnotation.listType()) - .repeatedGroup() - .optional(BINARY).as(stringType()).named("element") - .named("list") - .named("LIST[STRING]") - .named("root") + Types.buildMessage() + .optional(BINARY) + .as(stringType()) + .named("id1") + .optional(BINARY) + .as(stringType()) + .named( + "id2") + .optional(BOOLEAN) + .named("BOOLEAN") + .optional(INT32) + .as( + LogicalTypeAnnotation.intType(32, true)) + .named("INTEGER") + .optional(DOUBLE) + .named( + "NUMERIC") + .optional(INT32) + .as(LogicalTypeAnnotation.dateType()) + .named( + "DATE") + .optionalGroup() + .optional(INT32) + .as(LogicalTypeAnnotation.dateType()) + .named( + "min") + .optional(INT32) + .as(LogicalTypeAnnotation.dateType()) + .named("max") + .named( + "DATE_RANGE") + .optional(BINARY) + .as(stringType()) + .named("STRING") + .optional(INT32) + .as( + LogicalTypeAnnotation.intType(32, true)) + .named("MONEY") + .optionalGroup() + .as( + LogicalTypeAnnotation.listType()) + .repeatedGroup() + .optional(BOOLEAN) + .named( + "element") + .named("list") + .named("LIST[BOOLEAN]") + .optionalGroup() + .as( + LogicalTypeAnnotation.listType()) + .repeatedGroup() + .optionalGroup() + .optional( + INT32) + .as(LogicalTypeAnnotation.dateType()) + .named( + "min") + .optional(INT32) + .as( + LogicalTypeAnnotation.dateType()) + .named("max") + .named( + "element") + .named("list") + .named( + "LIST[DATE_RANGE]") + .optionalGroup() + .as( + LogicalTypeAnnotation.listType()) + .repeatedGroup() + .optional( + BINARY) + .as(stringType()) + .named( + "element") + .named("list") + .named( + "LIST[STRING]") + .named("root") ); } @@ -103,11 +161,12 @@ void writeAndRead() throws IOException { I18n.init(); // Prepare every input data - PrintSettings printSettings = new PrintSettings(false, - Locale.ROOT, - CONFIG, - (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), - (selectInfo) -> selectInfo.getSelect().getLabel() + PrintSettings printSettings = new PrintSettings( + false, + Locale.ROOT, + CONFIG, + (cer) -> EntityPrintId.from(cer.getEntityId(), cer.getEntityId()), + (selectInfo) -> selectInfo.getSelect().getLabel() ); // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized List results = getTestEntityResults(); @@ -116,7 +175,12 @@ void writeAndRead() throws IOException { // First we write to the buffer, than we read from it and parse it as TSV ByteArrayOutputStream output = new ByteArrayOutputStream(); - ParquetRenderer.writeToStream(output, getIdFields(), managedQuery.collectResultInfos(), printSettings, managedQuery.streamResults(OptionalLong.empty())); + ParquetRenderer.writeToStream( + output, + getIdFields(), + managedQuery.collectResultInfos(), + printSettings, + managedQuery.streamResults(OptionalLong.empty())); final byte[] buf = output.toByteArray(); @@ -138,7 +202,8 @@ void writeAndRead() throws IOException { log.info("\n{}", actual); - assertThat(actual).isEqualTo(ArrowResultGenerationTest.generateExpectedTSV(results, managedQuery.collectResultInfos())); + assertThat(actual).isEqualTo( + ArrowResultGenerationTest.generateExpectedTSV(results, managedQuery.collectResultInfos())); } @@ -167,8 +232,7 @@ private StringBuilder appendToString(StringBuilder builder, SimpleGroup group, b } builder.append("]"); - } - else if (field.isPrimitive()) { + } else if (field.isPrimitive()) { final String stringValue = group.getValueToString(fieldIndex, i); if (inSubGroup) { builder.append("\"").append(name).append("\":"); @@ -188,24 +252,20 @@ else if (field.isPrimitive()) { } } - } - catch (RuntimeException e) { + } catch (RuntimeException e) { // Trapped into an unset field if (!inSubGroup) { // Only print null if not in a struct (actual subgroup) builder.append("null"); - } - else if (fieldIndex > 0 && fieldIndex < fieldCount) { + } else if (fieldIndex > 0 && fieldIndex < fieldCount) { // Remove the last `,` builder.deleteCharAt(builder.length() - 1); } - } - finally { + } finally { if (fieldIndex + 1 < fieldCount) { if (inSubGroup) { builder.append(","); - } - else { + } else { builder.append("\t"); } } diff --git a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/CachedStoreTest.java b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/CachedStoreTest.java index 31036724c33..9c6c7798ef9 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/CachedStoreTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/CachedStoreTest.java @@ -16,20 +16,25 @@ public class CachedStoreTest { @RegisterExtension private final static XodusEnvironmentExtension ENVIRONMENT = new XodusEnvironmentExtension(); - private final XodusStore xodusStore = new XodusStore(ENVIRONMENT, getClass().getSimpleName(), (env) -> {}, (env) -> {}); + private final XodusStore xodusStore = new XodusStore(ENVIRONMENT, getClass().getSimpleName(), (env) -> { + }, (env) -> { + }); private final SerializingStore backingStore = new SerializingStore<>( - xodusStore, - null, - Jackson.MAPPER, - String.class, - String.class, - false, - false, - null, - new DirectExecutorService() + xodusStore, + null, + Jackson.MAPPER, + String.class, + String.class, + false, + false, + null, + new DirectExecutorService() ); - private final CachedStore cachedStore = new CachedStore<>(backingStore, CaffeineSpec.parse("softValues"), null); + private final CachedStore cachedStore = new CachedStore<>( + backingStore, + CaffeineSpec.parse("softValues"), + null); @BeforeEach public void beforeEach() { @@ -46,18 +51,22 @@ public void addAndRetrieve() { // Backing Store assertThat(backingStore.getAllKeys()).as("All expected keys in backing store") - .containsExactlyInAnyOrder(IntStream.range(0, 10).mapToObj("t1_%d"::formatted).toArray(String[]::new)); + .containsExactlyInAnyOrder( + IntStream.range(0, 10).mapToObj("t1_%d"::formatted).toArray(String[]::new)); assertThat(backingStore.getAll()).as("All expected values in backing store") - .containsExactlyInAnyOrder(IntStream.range(0, 10).mapToObj("%d"::formatted).toArray(String[]::new)); + .containsExactlyInAnyOrder( + IntStream.range(0, 10).mapToObj("%d"::formatted).toArray(String[]::new)); // Cached Store (getAllKeys only delegates) assertThat(cachedStore.getAllKeys()).as("All expected keys in cached store") - .containsExactlyInAnyOrder(IntStream.range(0, 10).mapToObj("t1_%d"::formatted).toArray(String[]::new)); + .containsExactlyInAnyOrder( + IntStream.range(0, 10).mapToObj("t1_%d"::formatted).toArray(String[]::new)); assertThat(cachedStore.getAll()).as("All expected values in cached store") - .containsExactlyInAnyOrder(IntStream.range(0, 10).mapToObj("%d"::formatted).toArray(String[]::new)); + .containsExactlyInAnyOrder( + IntStream.range(0, 10).mapToObj("%d"::formatted).toArray(String[]::new)); // Retrieve individually @@ -76,10 +85,12 @@ public void loadAndRetrieve() { // Cached Store assertThat(cachedStore.getAllKeys()).as("All expected keys in backing store") - .containsExactlyInAnyOrder(IntStream.range(0, 10).mapToObj("t2_%d"::formatted).toArray(String[]::new)); + .containsExactlyInAnyOrder( + IntStream.range(0, 10).mapToObj("t2_%d"::formatted).toArray(String[]::new)); assertThat(cachedStore.getAll()).as("All expected values in backing store") - .containsExactlyInAnyOrder(IntStream.range(0, 10).mapToObj("%d"::formatted).toArray(String[]::new)); + .containsExactlyInAnyOrder( + IntStream.range(0, 10).mapToObj("%d"::formatted).toArray(String[]::new)); // Retrieve individually @@ -100,7 +111,8 @@ public void add() { } assertThat(backingStore.getAll()).as("All expected values in backing store") - .containsExactlyInAnyOrder(IntStream.range(0, 10).mapToObj("%d"::formatted).toArray(String[]::new)); + .containsExactlyInAnyOrder( + IntStream.range(0, 10).mapToObj("%d"::formatted).toArray(String[]::new)); // Update values (should not succeed as keys are already present) for (int i : IntStream.range(0, 10).toArray()) { @@ -114,7 +126,8 @@ public void add() { } assertThat(backingStore.getAll()).as("All expected values in backing store") - .containsExactlyInAnyOrder(IntStream.range(0, 10).mapToObj("%d"::formatted).toArray(String[]::new)); + .containsExactlyInAnyOrder( + IntStream.range(0, 10).mapToObj("%d"::formatted).toArray(String[]::new)); } @Test @@ -129,7 +142,8 @@ public void update() { } assertThat(backingStore.getAll()).as("All expected values in backing store") - .containsExactlyInAnyOrder(IntStream.range(0, 10).mapToObj("%d"::formatted).toArray(String[]::new)); + .containsExactlyInAnyOrder( + IntStream.range(0, 10).mapToObj("%d"::formatted).toArray(String[]::new)); // Update values for (int i : IntStream.range(0, 10).toArray()) { @@ -142,7 +156,8 @@ public void update() { assertThat(cachedStore.get("t4_" + i)).isEqualTo("%d_updated".formatted(i)); assertThat(backingStore.getAll()).as("All expected values in backing store") - .containsExactlyInAnyOrder(IntStream.range(0, 10).mapToObj("%d_updated"::formatted).toArray(String[]::new)); + .containsExactlyInAnyOrder( + IntStream.range(0, 10).mapToObj("%d_updated"::formatted).toArray(String[]::new)); } // Check for equality @@ -159,10 +174,12 @@ public void remove() { assertThat(cachedStore.getAll()).as("All expected values in cached store") - .containsExactlyInAnyOrder(IntStream.range(0, 10).mapToObj("%d"::formatted).toArray(String[]::new)); + .containsExactlyInAnyOrder( + IntStream.range(0, 10).mapToObj("%d"::formatted).toArray(String[]::new)); assertThat(backingStore.getAll()).as("All expected values in backing store") - .containsExactlyInAnyOrder(IntStream.range(0, 10).mapToObj("%d"::formatted).toArray(String[]::new)); + .containsExactlyInAnyOrder( + IntStream.range(0, 10).mapToObj("%d"::formatted).toArray(String[]::new)); // Remove half of the entries for (int i : IntStream.range(0, 5).toArray()) { @@ -171,10 +188,12 @@ public void remove() { assertThat(cachedStore.getAll()).as("All expected values in cached store") - .containsExactlyInAnyOrder(IntStream.range(5, 10).mapToObj("%d"::formatted).toArray(String[]::new)); + .containsExactlyInAnyOrder( + IntStream.range(5, 10).mapToObj("%d"::formatted).toArray(String[]::new)); assertThat(backingStore.getAll()).as("All expected values in backing store") - .containsExactlyInAnyOrder(IntStream.range(5, 10).mapToObj("%d"::formatted).toArray(String[]::new)); + .containsExactlyInAnyOrder( + IntStream.range(5, 10).mapToObj("%d"::formatted).toArray(String[]::new)); // Check retrieval of deleted values for (int i : IntStream.range(0, 5).toArray()) { diff --git a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java index 8c370116701..3f123b9384e 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java @@ -3,10 +3,10 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; +import jakarta.validation.Validator; import java.io.File; import java.io.IOException; import java.util.concurrent.Executors; -import jakarta.validation.Validator; import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.Query; @@ -42,7 +42,13 @@ public class SerializingStoreDumpTest { private static final MetaStorage STORAGE = new NonPersistentStoreFactory().createMetaStorage(); private static final NamespacedStorageProvider NAMESPACED_STORAGE_PROVIDER = new TestNamespacedStorageProvider(); // Test data - private final ManagedQuery managedQuery = new ManagedQuery(mock(Query.class), new UserId("test"), new DatasetId("dataset"), STORAGE, null, null); + private final ManagedQuery managedQuery = new ManagedQuery( + mock(Query.class), + new UserId("test"), + new DatasetId("dataset"), + STORAGE, + null, + null); private final ConceptQuery cQuery = new ConceptQuery(new CQReusedQuery(managedQuery.getId())); private final User user = new User("username", "userlabel", STORAGE); @@ -93,10 +99,10 @@ public void testCorruptValueDump() throws IOException { // Open that store again, with a different config to insert a corrupt entry // (UserId & ManagedQuery) SerializingStore store = createSerializedStore( - config, - env, - Validators.newValidator(), - new StoreInfo<>(info.getName(), UserId.class, QueryDescription.class) + config, + env, + Validators.newValidator(), + new StoreInfo<>(info.getName(), UserId.class, QueryDescription.class) ); store.add(invalidId, cQuery); } @@ -116,43 +122,48 @@ public void testCorruptValueDump() throws IOException { assertThat(result).isEqualTo(expectedResult); } - final File[] dumpFiles = SerializingStore.makeDumpFileName(invalidId.toString(), tmpDir, info.getName()).getParentFile() - .listFiles(file -> file.getName().endsWith(SerializingStore.DUMP_FILE_EXTENSION)); + final File[] dumpFiles = SerializingStore.makeDumpFileName( + invalidId.toString(), + tmpDir, + info.getName()) + .getParentFile() + .listFiles( + file -> file.getName().endsWith(SerializingStore.DUMP_FILE_EXTENSION)); - final File[] exceptionFiles = SerializingStore.makeDumpFileName(invalidId.toString(), tmpDir, info.getName()).getParentFile() - .listFiles(file -> file.getName().endsWith(SerializingStore.EXCEPTION_FILE_EXTENSION)); + final File[] exceptionFiles = SerializingStore.makeDumpFileName( + invalidId.toString(), + tmpDir, + info.getName()) + .getParentFile() + .listFiles( + file -> file.getName().endsWith(SerializingStore.EXCEPTION_FILE_EXTENSION)); - assertThat(dumpFiles) - .isNotNull() - .hasSize(1); + assertThat(dumpFiles).isNotNull().hasSize(1); - assertThat(exceptionFiles) - .isNotNull() - .hasSize(1); + assertThat(exceptionFiles).isNotNull().hasSize(1); assertThat(stringMapper.readValue(dumpFiles[0], QueryDescription.class)).isEqualTo(cQuery); } private SerializingStore createSerializedStore( - XodusStoreFactory config, - Environment environment, - Validator validator, - StoreInfo storeId) { + XodusStoreFactory config, + Environment environment, + Validator validator, + StoreInfo storeId) { return new SerializingStore<>( - new XodusStore(environment, storeId.getName(), - (e) -> { - }, - (e) -> { - } - ), - validator, - binaryMapper, - storeId.getKeyType(), - storeId.getValueType(), - config.isValidateOnWrite(), - config.isRemoveUnreadableFromStore(), - config.getUnreadableDataDumpDirectory(), Executors.newSingleThreadExecutor() + new XodusStore(environment, storeId.getName(), (e) -> { + }, (e) -> { + } + ), + validator, + binaryMapper, + storeId.getKeyType(), + storeId.getValueType(), + config.isValidateOnWrite(), + config.isRemoveUnreadableFromStore(), + config.getUnreadableDataDumpDirectory(), + Executors.newSingleThreadExecutor() ); } @@ -176,10 +187,10 @@ public void testCorruptKeyDump() throws IOException { // Open that store again, with a different config to insert a corrupt entry // (String & ManagedQuery) SerializingStore store = createSerializedStore( - config, - env, - Validators.newValidator(), - new StoreInfo<>(info.getName(), String.class, QueryDescription.class) + config, + env, + Validators.newValidator(), + new StoreInfo<>(info.getName(), String.class, QueryDescription.class) ); store.add(invalidId, cQuery); } @@ -201,19 +212,25 @@ public void testCorruptKeyDump() throws IOException { // Test if the correct number of dumpfiles was generated // The file name is bogus since the id wasn't read properly - final File[] dumpFiles = SerializingStore.makeDumpFileName(invalidId, tmpDir, info.getName()).getParentFile() - .listFiles(file -> file.getName().endsWith(SerializingStore.DUMP_FILE_EXTENSION)); + final File[] dumpFiles = SerializingStore.makeDumpFileName( + invalidId, + tmpDir, + info.getName()) + .getParentFile() + .listFiles( + file -> file.getName().endsWith(SerializingStore.DUMP_FILE_EXTENSION)); - final File[] exceptionFiles = SerializingStore.makeDumpFileName(invalidId, tmpDir, info.getName()).getParentFile() - .listFiles(file -> file.getName().endsWith(SerializingStore.EXCEPTION_FILE_EXTENSION)); + final File[] exceptionFiles = SerializingStore.makeDumpFileName( + invalidId, + tmpDir, + info.getName()) + .getParentFile() + .listFiles( + file -> file.getName().endsWith(SerializingStore.EXCEPTION_FILE_EXTENSION)); - assertThat(dumpFiles) - .isNotNull() - .hasSize(1); + assertThat(dumpFiles).isNotNull().hasSize(1); - assertThat(exceptionFiles) - .isNotNull() - .hasSize(1); + assertThat(exceptionFiles).isNotNull().hasSize(1); assertThat(stringMapper.readValue(dumpFiles[0], QueryDescription.class)).isEqualTo(cQuery); @@ -231,7 +248,11 @@ public void testCorruptionRemoval() { { // Open a store and insert a valid key-value pair (UserId & User) - SerializingStore store = createSerializedStore(config, env, Validators.newValidator(), USER_STORE_ID); + SerializingStore store = createSerializedStore( + config, + env, + Validators.newValidator(), + USER_STORE_ID); store.add(new UserId("testU1"), user); } @@ -239,20 +260,20 @@ public void testCorruptionRemoval() { // corrupt value { SerializingStore store = createSerializedStore( - config, - env, - Validators.newValidator(), - new StoreInfo<>(USER_STORE_ID.getName(), String.class, QueryDescription.class) + config, + env, + Validators.newValidator(), + new StoreInfo<>(USER_STORE_ID.getName(), String.class, QueryDescription.class) ); store.add("not a valid conquery Id", cQuery); } { SerializingStore store = createSerializedStore( - config, - env, - Validators.newValidator(), - new StoreInfo<>(USER_STORE_ID.getName(), UserId.class, QueryDescription.class) + config, + env, + Validators.newValidator(), + new StoreInfo<>(USER_STORE_ID.getName(), UserId.class, QueryDescription.class) ); store.add(new UserId("testU2"), cQuery); } @@ -261,7 +282,11 @@ public void testCorruptionRemoval() { { // Reopen the store with correct configuration and try to iterate over all // entries (this triggers the dump or removal of invalid entries) - SerializingStore store = createSerializedStore(config, env, Validators.newValidator(), USER_STORE_ID); + SerializingStore store = createSerializedStore( + config, + env, + Validators.newValidator(), + USER_STORE_ID); IterationStatistic expectedResult = new IterationStatistic(); expectedResult.setTotalProcessed(3); expectedResult.setFailedKeys(1); @@ -275,7 +300,11 @@ public void testCorruptionRemoval() { { // Reopen again to check that the corrupted values have been removed previously - SerializingStore store = createSerializedStore(config, env, Validators.newValidator(), USER_STORE_ID); + SerializingStore store = createSerializedStore( + config, + env, + Validators.newValidator(), + USER_STORE_ID); IterationStatistic expectedResult = new IterationStatistic(); expectedResult.setTotalProcessed(1); expectedResult.setFailedKeys(0); diff --git a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java index 6bf2d30ce4e..0487f7c6037 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java @@ -3,6 +3,7 @@ import static com.bakdata.conquery.util.SerialisationObjectsUtil.*; import static org.assertj.core.api.Assertions.assertThat; +import jakarta.validation.Validator; import java.io.IOException; import java.time.LocalDate; import java.time.ZonedDateTime; @@ -15,7 +16,6 @@ import java.util.Random; import java.util.UUID; import java.util.stream.Stream; -import jakarta.validation.Validator; import com.bakdata.conquery.apiv1.IdLabel; import com.bakdata.conquery.apiv1.MeProcessor; @@ -112,25 +112,25 @@ public class SerializationTests extends AbstractSerializationTest { public static Stream> rangeData() { final int SEED = 7; Random random = new Random(SEED); - return Stream - .generate(() -> { - int first = random.nextInt(); - int second = random.nextInt(); - - if (first < second) { - return Range.of(first, second); - } - return Range.of(second, first); - }) - .filter(Range::isOrdered) - .flatMap(range -> Stream.of( - range, - Range.exactly(range.getMin()), - Range.atMost(range.getMin()), - Range.atLeast(range.getMin()) + return Stream.generate(() -> { + int first = random.nextInt(); + int second = random.nextInt(); + + if (first < second) { + return Range.of(first, second); + } + return Range.of(second, first); + }) + .filter(Range::isOrdered) + .flatMap( + range -> Stream.of( + range, + Range.exactly(range.getMin()), + Range.atMost(range.getMin()), + Range.atLeast(range.getMin()) )) - .filter(Range::isOrdered) - .limit(100); + .filter(Range::isOrdered) + .limit(100); } @Test @@ -140,31 +140,33 @@ public void dataset() throws IOException, JSONException { dataset.setLabel("Dataset"); dataset.setStorageProvider(getDatasetRegistry()); - SerializationTestUtil - .forType(Dataset.class) - .objectMappers(getManagerInternalMapper(), getInternalCommunicationMapper()) - .customizingAssertion(assertion -> assertion.ignoringFields("storageProvider")) - .test(dataset); + SerializationTestUtil.forType(Dataset.class) + .objectMappers( + getManagerInternalMapper(), + getInternalCommunicationMapper()) + .customizingAssertion( + assertion -> assertion.ignoringFields("storageProvider")) + .test(dataset); } @Test public void passwordCredential() throws IOException, JSONException { PasswordCredential credential = new PasswordCredential("testPassword"); - SerializationTestUtil - .forType(PasswordCredential.class) - .objectMappers(getManagerInternalMapper()) - .test(credential); + SerializationTestUtil.forType(PasswordCredential.class) + .objectMappers(getManagerInternalMapper()) + .test( + credential); } @Test public void role() throws IOException, JSONException { Role mandator = new Role("company", "company", getMetaStorage()); - SerializationTestUtil - .forType(Role.class) - .objectMappers(getManagerInternalMapper(), getApiMapper()) - .test(mandator); + SerializationTestUtil.forType(Role.class) + .objectMappers(getManagerInternalMapper(), getApiMapper()) + .test( + mandator); } /* @@ -175,25 +177,31 @@ public void user() throws IOException, JSONException { User user = new User("user", "user", getMetaStorage()); user.setMetaStorage(getMetaStorage()); user.addPermission(DatasetPermission.onInstance(Ability.READ, new DatasetId("test"))); - user.addPermission(ExecutionPermission.onInstance(Ability.READ, new ManagedExecutionId(new DatasetId("dataset"), UUID.randomUUID()))); + user.addPermission( + ExecutionPermission.onInstance( + Ability.READ, + new ManagedExecutionId(new DatasetId("dataset"), UUID.randomUUID()))); Role role = new Role("company", "company", getMetaStorage()); getMetaStorage().addRole(role); user.addRole(role.getId()); - SerializationTestUtil - .forType(User.class) - .objectMappers(getManagerInternalMapper(), getApiMapper()) - .injectables(getMetaStorage()) - .test(user); + SerializationTestUtil.forType(User.class) + .objectMappers(getManagerInternalMapper(), getApiMapper()) + .injectables( + getMetaStorage()) + .test(user); } @Test public void group() throws IOException, JSONException { Group group = new Group("group", "group", getMetaStorage()); group.addPermission(DatasetPermission.onInstance(Ability.READ, new DatasetId("test"))); - group.addPermission(ExecutionPermission.onInstance(Ability.READ, new ManagedExecutionId(new DatasetId("dataset"), UUID.randomUUID()))); + group.addPermission( + ExecutionPermission.onInstance( + Ability.READ, + new ManagedExecutionId(new DatasetId("dataset"), UUID.randomUUID()))); Role role = new Role("company", "company", getMetaStorage()); RoleId roleId = role.getId(); @@ -208,10 +216,10 @@ public void group() throws IOException, JSONException { metaStorage.addRole(role); metaStorage.addUser(user); - SerializationTestUtil - .forType(Group.class) - .objectMappers(getManagerInternalMapper(), getApiMapper()) - .test(group); + SerializationTestUtil.forType(Group.class) + .objectMappers(getManagerInternalMapper(), getApiMapper()) + .test( + group); } @Test @@ -248,15 +256,22 @@ public void bucketCompoundDateRange() throws JSONException, IOException { Import imp = new Import("importTest", table.getId()); - CompoundDateRangeStore compoundStore = - new CompoundDateRangeStore(startCol.getName(), endCol.getName(), new BitSetStore(BitSet.valueOf(new byte[]{0b1000}), new BitSet(), 4)); + CompoundDateRangeStore compoundStore = new CompoundDateRangeStore( + startCol.getName(), + endCol.getName(), + new BitSetStore(BitSet.valueOf(new byte[]{0b1000}), new BitSet(), 4)); //0b1000 is a binary representation of 8 so that the 4th is set to make sure that BitSet length is 4. ColumnStore startStore = new IntegerDateStore(new ShortArrayStore(new short[]{1, 2, 3, 4}, Short.MIN_VALUE)); ColumnStore endStore = new IntegerDateStore(new ShortArrayStore(new short[]{5, 6, 7, 8}, Short.MIN_VALUE)); - Bucket bucket = - new Bucket(0, Object2IntMaps.singleton("0", 0), Object2IntMaps.singleton("0", 4), 4, imp.getId(), new ColumnStore[]{startStore, endStore, compoundStore}); + Bucket bucket = new Bucket( + 0, + Object2IntMaps.singleton("0", 0), + Object2IntMaps.singleton("0", 4), + 4, + imp.getId(), + new ColumnStore[]{startStore, endStore, compoundStore}); compoundStore.setParent(bucket); @@ -267,12 +282,14 @@ public void bucketCompoundDateRange() throws JSONException, IOException { final Validator validator = Validators.newValidator(); - SerializationTestUtil - .forType(Bucket.class) - .objectMappers(getInternalCommunicationMapper()) - .injectables(values -> values.add(Validator.class, validator) - .add(NamespacedStorageProvider.class, getShardNamespacedStorageProvider())) - .test(bucket); + SerializationTestUtil.forType(Bucket.class) + .objectMappers(getInternalCommunicationMapper()) + .injectables( + values -> values.add(Validator.class, validator) + .add( + NamespacedStorageProvider.class, + getShardNamespacedStorageProvider())) + .test(bucket); } @@ -284,10 +301,12 @@ public void table() throws JSONException, IOException { Table table = getTable(getNamespaceStorage()); - SerializationTestUtil - .forType(Table.class) - .objectMappers(getNamespacePersistenceMapper(), getApiMapper(), getInternalCommunicationMapper()) - .test(table); + SerializationTestUtil.forType(Table.class) + .objectMappers( + getNamespacePersistenceMapper(), + getApiMapper(), + getInternalCommunicationMapper()) + .test(table); } { @@ -298,10 +317,7 @@ public void table() throws JSONException, IOException { getWorkerStorage().addTable(table); - SerializationTestUtil - .forType(Table.class) - .objectMappers(getWorkerPersistenceMapper()) - .test(table); + SerializationTestUtil.forType(Table.class).objectMappers(getWorkerPersistenceMapper()).test(table); } } @@ -325,15 +341,17 @@ public void table() throws JSONException, IOException { @Test public void filterValueMoneyRange() throws JSONException, IOException { - FilterValue.CQMoneyRangeFilter filterValue = - new FilterValue.CQMoneyRangeFilter(FilterId.Parser.INSTANCE.parse("dataset.concept.connector.filter"), new Range.LongRange(2000L, 30000L)); + FilterValue.CQMoneyRangeFilter filterValue = new FilterValue.CQMoneyRangeFilter( + FilterId.Parser.INSTANCE.parse("dataset.concept.connector.filter"), + new Range.LongRange(2000L, 30000L)); filterValue.setConfig(getConfig()); - SerializationTestUtil - .forType(FilterValue.class) - .objectMappers(getNamespacePersistenceMapper(), getApiMapper()) - .test(filterValue); + SerializationTestUtil.forType(FilterValue.class) + .objectMappers( + getNamespacePersistenceMapper(), + getApiMapper()) + .test(filterValue); } @Test @@ -343,11 +361,12 @@ public void treeConcept() throws IOException, JSONException { final Dataset dataset = createDataset(getNamespaceStorage(), getDatasetRegistry()); TreeConcept concept = createConcept(dataset, getNamespaceStorage()); - SerializationTestUtil - .forType(Concept.class) - .objectMappers(getNamespacePersistenceMapper(), getApiMapper()) - .injectables(dataset, getDatasetRegistry()) - .test(concept); + SerializationTestUtil.forType(Concept.class) + .objectMappers( + getNamespacePersistenceMapper(), + getApiMapper()) + .injectables(dataset, getDatasetRegistry()) + .test(concept); } { @@ -355,10 +374,7 @@ public void treeConcept() throws IOException, JSONException { final Dataset dataset = createDataset(getWorkerStorage(), getShardNamespacedStorageProvider()); TreeConcept concept = createConcept(dataset, getWorkerStorage()); - SerializationTestUtil - .forType(Concept.class) - .objectMappers(getInternalCommunicationMapper()) - .test(concept); + SerializationTestUtil.forType(Concept.class).objectMappers(getInternalCommunicationMapper()).test(concept); } } @@ -368,8 +384,10 @@ public void persistentIdMap() throws JSONException, IOException { EntityIdMap persistentMap = IdMapSerialisationTest.createTestPersistentMap(getNamespaceStorage()); SerializationTestUtil.forType(EntityIdMap.class) - .objectMappers(getNamespacePersistenceMapper(), getApiMapper()) - .test(persistentMap); + .objectMappers( + getNamespacePersistenceMapper(), + getApiMapper()) + .test(persistentMap); } @@ -388,10 +406,10 @@ public void formConfig() throws JSONException, IOException { FormConfig formConfig = new FormConfig(form.getClass().getAnnotation(CPSType.class).id(), values); formConfig.setDataset(dataset.getId()); - SerializationTestUtil - .forType(FormConfig.class) - .objectMappers(getManagerInternalMapper(), getApiMapper()) - .test(formConfig); + SerializationTestUtil.forType(FormConfig.class) + .objectMappers(getManagerInternalMapper(), getApiMapper()) + .test( + formConfig); } @Test @@ -404,16 +422,24 @@ public void managedQuery() throws JSONException, IOException { getNamespaceStorage().updateDataset(dataset); getMetaStorage().updateUser(user); - ManagedQuery execution = new ManagedQuery(null, user.getId(), dataset.getId(), getMetaStorage(), getDatasetRegistry(), getConfig()); + ManagedQuery execution = new ManagedQuery( + null, + user.getId(), + dataset.getId(), + getMetaStorage(), + getDatasetRegistry(), + getConfig()); execution.setTags(new String[]{"test-tag"}); // Trigger UUID creation execution.getId(); SerializationTestUtil.forType(ManagedExecution.class) - .objectMappers(getManagerInternalMapper(), getApiMapper()) - .injectables(getMetaStorage()) - .test(execution); + .objectMappers( + getManagerInternalMapper(), + getApiMapper()) + .injectables(getMetaStorage()) + .test(execution); } @Test @@ -424,9 +450,11 @@ public void testExportForm() throws JSONException, IOException { final ExportForm exportForm = createExportForm(dataset, getNamespaceStorage()); SerializationTestUtil.forType(QueryDescription.class) - .objectMappers(getManagerInternalMapper(), getApiMapper()) - .checkHashCode() - .test(exportForm); + .objectMappers( + getManagerInternalMapper(), + getApiMapper()) + .checkHashCode() + .test(exportForm); } @Test @@ -437,17 +465,24 @@ public void managedForm() throws JSONException, IOException { final User user = createUser(getMetaStorage()); final ExportForm exportForm = createExportForm(dataset, getNamespaceStorage()); - ManagedInternalForm execution = - new ManagedInternalForm<>(exportForm, user.getId(), dataset.getId(), getMetaStorage(), getDatasetRegistry(), getConfig()); + ManagedInternalForm execution = new ManagedInternalForm<>( + exportForm, + user.getId(), + dataset.getId(), + getMetaStorage(), + getDatasetRegistry(), + getConfig()); execution.setTags(new String[]{"test-tag"}); // Trigger UUID creation execution.getId(); SerializationTestUtil.forType(ManagedExecution.class) - .objectMappers(getManagerInternalMapper(), getApiMapper()) - .injectables(getDatasetRegistry()) - .test(execution); + .objectMappers( + getManagerInternalMapper(), + getApiMapper()) + .injectables(getDatasetRegistry()) + .test(execution); } @Test @@ -455,25 +490,34 @@ public void testExternalExecution() throws IOException, JSONException { final String subType = "test-type"; JsonNodeFactory factory = new JsonNodeFactory(false); - ObjectNode node = new ObjectNode(factory, - Map.of( - "some-other-member", new TextNode("some-other-value") - ) + ObjectNode node = new ObjectNode( + factory, + Map.of( + "some-other-member", + new TextNode("some-other-value") + ) ); ExternalForm form = new ExternalForm(node, subType); final Dataset dataset = createDataset(getNamespaceStorage(), getDatasetRegistry()); final User user = createUser(getMetaStorage()); - final ExternalExecution execution = new ExternalExecution(form, user.getId(), dataset.getId(), getMetaStorage(), getDatasetRegistry(), getConfig()); + final ExternalExecution execution = new ExternalExecution( + form, + user.getId(), + dataset.getId(), + getMetaStorage(), + getDatasetRegistry(), + getConfig()); // Trigger UUID creation execution.getId(); SerializationTestUtil.forType(ManagedExecution.class) - .objectMappers(getManagerInternalMapper()) - .injectables(getDatasetRegistry()) - .test(execution); + .objectMappers(getManagerInternalMapper()) + .injectables( + getDatasetRegistry()) + .test(execution); } @@ -483,25 +527,28 @@ public void cqConcept() throws Exception { // Manager final CQConcept cqConcept = createCqConcept(getNamespaceStorage(), getDatasetRegistry()); - SerializationTestUtil - .forType(CQConcept.class) - .objectMappers(getManagerInternalMapper(), getApiMapper()) - .test(cqConcept); + SerializationTestUtil.forType(CQConcept.class) + .objectMappers( + getManagerInternalMapper(), + getApiMapper()) + .test(cqConcept); } { // Shard final CQConcept cqConcept = createCqConcept(getWorkerStorage(), getShardNamespacedStorageProvider()); - SerializationTestUtil - .forType(CQConcept.class) - .objectMappers(getInternalCommunicationMapper()) - .test(cqConcept); + SerializationTestUtil.forType(CQConcept.class) + .objectMappers(getInternalCommunicationMapper()) + .test( + cqConcept); } } @NotNull - private static CQConcept createCqConcept(NamespacedStorage namespaceStorage, NamespacedStorageProvider storageProvider) throws Exception { + private static CQConcept createCqConcept( + NamespacedStorage namespaceStorage, + NamespacedStorageProvider storageProvider) throws Exception { Dataset dataset = createDataset(namespaceStorage, storageProvider); final TreeConcept concept = new TreeConcept(); @@ -536,31 +583,39 @@ public void executionCreationPlanError() throws JSONException, IOException { ConqueryError error = new ConqueryError.ExecutionProcessingError(); - SerializationTestUtil - .forType(ConqueryError.class) - .objectMappers(getManagerInternalMapper(), getInternalCommunicationMapper(), getApiMapper()) - .test(error); + SerializationTestUtil.forType(ConqueryError.class) + .objectMappers( + getManagerInternalMapper(), + getInternalCommunicationMapper(), + getApiMapper()) + .test(error); } @Test public void executionCreationResolveError() throws JSONException, IOException { ConqueryError error = new ConqueryError.ExecutionCreationResolveError(new DatasetId("test")); - SerializationTestUtil - .forType(ConqueryError.class) - .objectMappers(getManagerInternalMapper(), getInternalCommunicationMapper(), getApiMapper()) - .test(error); + SerializationTestUtil.forType(ConqueryError.class) + .objectMappers( + getManagerInternalMapper(), + getInternalCommunicationMapper(), + getApiMapper()) + .test(error); } @Test public void executionQueryJobError() throws JSONException, IOException { log.info("Beware, this test will print an ERROR message."); - ConqueryError error = new ConqueryError.ExecutionJobErrorWrapper(new Entity("5"), new ConqueryError.UnknownError(null)); + ConqueryError error = new ConqueryError.ExecutionJobErrorWrapper( + new Entity("5"), + new ConqueryError.UnknownError(null)); - SerializationTestUtil - .forType(ConqueryError.class) - .objectMappers(getManagerInternalMapper(), getInternalCommunicationMapper(), getApiMapper()) - .test(error); + SerializationTestUtil.forType(ConqueryError.class) + .objectMappers( + getManagerInternalMapper(), + getInternalCommunicationMapper(), + getApiMapper()) + .test(error); } @Test @@ -568,18 +623,21 @@ public void meInformation() throws IOException, JSONException { User user = new User("name", "labe", getMetaStorage()); MeProcessor.FrontendMeInformation info = MeProcessor.FrontendMeInformation.builder() - .userName(user.getLabel()) - .hideLogoutButton(false) - .groups(List.of(new IdLabel<>(new GroupId("test_group"), "test_group_label"))) - .datasetAbilities(Map.of(new DatasetId("testdataset"), - new MeProcessor.FrontendDatasetAbility(true, true, true) - )) - .build(); + .userName( + user.getLabel()) + .hideLogoutButton(false) + .groups( + List.of(new IdLabel<>(new GroupId("test_group"), "test_group_label"))) + .datasetAbilities( + Map.of(new DatasetId("testdataset"), new MeProcessor.FrontendDatasetAbility(true, true, true) + )) + .build(); - SerializationTestUtil - .forType(MeProcessor.FrontendMeInformation.class) - .objectMappers(getManagerInternalMapper(), getApiMapper()) - .test(info); + SerializationTestUtil.forType(MeProcessor.FrontendMeInformation.class) + .objectMappers( + getManagerInternalMapper(), + getApiMapper()) + .test(info); } @Test @@ -619,13 +677,13 @@ public void testFormQuery() throws Exception { features.setChildren(Collections.singletonList(concept)); AbsoluteFormQuery query = new AbsoluteFormQuery( - subQuery, - CDateRange.exactly(LocalDate.now()).toSimpleRange(), - ArrayConceptQuery.createFromFeatures(Collections.singletonList(features)), - List.of( - ExportForm.ResolutionAndAlignment.of(Resolution.COMPLETE, Alignment.NO_ALIGN), - ExportForm.ResolutionAndAlignment.of(Resolution.QUARTERS, Alignment.QUARTER) - ) + subQuery, + CDateRange.exactly(LocalDate.now()).toSimpleRange(), + ArrayConceptQuery.createFromFeatures(Collections.singletonList(features)), + List.of( + ExportForm.ResolutionAndAlignment.of(Resolution.COMPLETE, Alignment.NO_ALIGN), + ExportForm.ResolutionAndAlignment.of(Resolution.QUARTERS, Alignment.QUARTER) + ) ); getNamespaceStorage().updateDataset(dataset); @@ -635,10 +693,13 @@ public void testFormQuery() throws Exception { workerStorage.updateDataset(dataset); workerStorage.updateConcept(testConcept); - SerializationTestUtil - .forType(AbsoluteFormQuery.class) - .objectMappers(getManagerInternalMapper(), getInternalCommunicationMapper(), getApiMapper(), getWorkerPersistenceMapper()) - .test(query); + SerializationTestUtil.forType(AbsoluteFormQuery.class) + .objectMappers( + getManagerInternalMapper(), + getInternalCommunicationMapper(), + getApiMapper(), + getWorkerPersistenceMapper()) + .test(query); } @Test @@ -670,15 +731,24 @@ public void cBlock() throws Exception { getWorkerStorage().updateConcept(concept); getWorkerStorage().addImport(imp); - final Bucket bucket = new Bucket(0, Object2IntMaps.emptyMap(), Object2IntMaps.emptyMap(), 0, imp.getId(), new ColumnStore[0]); + final Bucket bucket = new Bucket( + 0, + Object2IntMaps.emptyMap(), + Object2IntMaps.emptyMap(), + 0, + imp.getId(), + new ColumnStore[0]); getWorkerStorage().addBucket(bucket); - final CBlock cBlock = new CBlock(bucket.getId(), connector.getId(), Collections.emptyMap(), Collections.emptyMap(), new int[0][]); + final CBlock cBlock = new CBlock( + bucket.getId(), + connector.getId(), + Collections.emptyMap(), + Collections.emptyMap(), + new int[0][]); - SerializationTestUtil.forType(CBlock.class) - .objectMappers(getWorkerPersistenceMapper()) - .test(cBlock); + SerializationTestUtil.forType(CBlock.class).objectMappers(getWorkerPersistenceMapper()).test(cBlock); } @Test @@ -686,64 +756,78 @@ public void testBiMapSerialization() throws JSONException, IOException { BiMap map = HashBiMap.create(); map.put("a", "1"); map.put("b", "2"); - SerializationTestUtil - .forType(new TypeReference>() { - }) - .objectMappers(getApiMapper(), getManagerInternalMapper()) - .test(map); + SerializationTestUtil.forType(new TypeReference>() { + }).objectMappers(getApiMapper(), getManagerInternalMapper()).test(map); } @Test public void testNonStrictNumbers() throws JSONException, IOException { SerializationTestUtil.forType(Double.class) - .objectMappers(getApiMapper(), getManagerInternalMapper()).test(Double.NaN, null); + .objectMappers(getApiMapper(), getManagerInternalMapper()) + .test( + Double.NaN, + null); SerializationTestUtil.forType(Double.class) - .objectMappers(getApiMapper(), getManagerInternalMapper()).test(Double.NEGATIVE_INFINITY, null); + .objectMappers(getApiMapper(), getManagerInternalMapper()) + .test( + Double.NEGATIVE_INFINITY, + null); SerializationTestUtil.forType(Double.class) - .objectMappers(getApiMapper(), getManagerInternalMapper()).test(Double.POSITIVE_INFINITY, null); + .objectMappers(getApiMapper(), getManagerInternalMapper()) + .test( + Double.POSITIVE_INFINITY, + null); SerializationTestUtil.forType(Double.class) - .objectMappers(getApiMapper(), getManagerInternalMapper()).test(Double.MAX_VALUE); + .objectMappers(getApiMapper(), getManagerInternalMapper()) + .test( + Double.MAX_VALUE); SerializationTestUtil.forType(Double.class) - .objectMappers(getApiMapper(), getManagerInternalMapper()).test(Double.MIN_VALUE); - SerializationTestUtil - .forType(EntityResult.class) - .objectMappers(getApiMapper(), getManagerInternalMapper()) - .test( - new MultilineEntityResult("4", List.of( - new Object[]{0, 1, 2}, - new Object[]{Double.NaN, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY} - ) - ), - new MultilineEntityResult("4", List.of( - new Object[]{0, 1, 2}, - new Object[]{null, null, null} - ) - ) - ); + .objectMappers(getApiMapper(), getManagerInternalMapper()) + .test( + Double.MIN_VALUE); + SerializationTestUtil.forType(EntityResult.class) + .objectMappers( + getApiMapper(), + getManagerInternalMapper()) + .test( + new MultilineEntityResult( + "4", + List.of( + new Object[]{0, 1, 2}, + new Object[]{Double.NaN, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY} + ) + ), + new MultilineEntityResult( + "4", + List.of( + new Object[]{0, 1, 2}, + new Object[]{null, null, null} + ) + ) + ); } @ParameterizedTest @MethodSource("rangeData") public void test(Range range) throws IOException, JSONException { - SerializationTestUtil - .forType(new TypeReference>() { - }) - .objectMappers(getApiMapper(), getManagerInternalMapper(), getInternalCommunicationMapper()) - .test(range); + SerializationTestUtil.forType(new TypeReference>() { + }).objectMappers(getApiMapper(), getManagerInternalMapper(), getInternalCommunicationMapper()).test(range); } @Test public void locale() throws JSONException, IOException { SerializationTestUtil.forType(Locale.class) - .objectMappers(getApiMapper(), getManagerInternalMapper()) - .test(Locale.GERMANY); + .objectMappers(getApiMapper(), getManagerInternalMapper()) + .test( + Locale.GERMANY); } @Test public void localeArray() throws JSONException, IOException { SerializationTestUtil.forType(Locale[].class) - .objectMappers(getApiMapper(), getManagerInternalMapper()) - .test(new Locale[]{Locale.GERMANY, Locale.ROOT, Locale.ENGLISH, Locale.US, Locale.UK}); + .objectMappers(getApiMapper(), getManagerInternalMapper()) + .test( + new Locale[]{Locale.GERMANY, Locale.ROOT, Locale.ENGLISH, Locale.US, Locale.UK}); } /** @@ -765,8 +849,10 @@ public void externalForm() throws IOException, JSONException { ExternalForm externalForm = getApiMapper().readerFor(QueryDescription.class).readValue(externalFormString); SerializationTestUtil.forType(QueryDescription.class) - .objectMappers(getApiMapper(), getManagerInternalMapper()) - .test(externalForm); + .objectMappers( + getApiMapper(), + getManagerInternalMapper()) + .test(externalForm); } /** @@ -800,8 +886,10 @@ public void externalFormArray() throws IOException, JSONException { ExternalForm externalForm = getApiMapper().readerFor(QueryDescription.class).readValue(externalFormString); ExternalForm externalForm2 = getApiMapper().readerFor(QueryDescription.class).readValue(externalFormString2); SerializationTestUtil.forType(QueryDescription[].class) - .objectMappers(getApiMapper(), getManagerInternalMapper()) - .test(new QueryDescription[]{externalForm, externalForm2}); + .objectMappers( + getApiMapper(), + getManagerInternalMapper()) + .test(new QueryDescription[]{externalForm, externalForm2}); } @@ -846,10 +934,14 @@ public void object2IntEmpty() throws JSONException, IOException { Object2IntMap empty = Object2IntMaps.emptyMap(); SerializationTestUtil.forType(new TypeReference>() { - }) - .objectMappers(getApiMapper(), getInternalCommunicationMapper(), getManagerInternalMapper()) - .customizingAssertion(RecursiveComparisonAssert::ignoringCollectionOrder) - .test(empty); + }) + .objectMappers( + getApiMapper(), + getInternalCommunicationMapper(), + getManagerInternalMapper()) + .customizingAssertion(RecursiveComparisonAssert::ignoringCollectionOrder) + .test( + empty); } @@ -862,44 +954,60 @@ public void object2IntString() throws JSONException, IOException { map.put("one", 1); map.put("two", 2); SerializationTestUtil.forType(new TypeReference>() { - }) - .objectMappers(getApiMapper(), getInternalCommunicationMapper(), getManagerInternalMapper()) - .customizingAssertion(RecursiveComparisonAssert::ignoringCollectionOrder) - .test(map); + }) + .objectMappers( + getApiMapper(), + getInternalCommunicationMapper(), + getManagerInternalMapper()) + .customizingAssertion(RecursiveComparisonAssert::ignoringCollectionOrder) + .test( + map); } @Test @Tag("OBJECT_2_INT_MAP") public void arrayObject2Int() throws JSONException, IOException { - Object2IntMap[] map = new Object2IntOpenHashMap[]{ - new Object2IntOpenHashMap<>() {{ - put("zero", 0); - }}, - new Object2IntOpenHashMap<>() {{ - put("zero", 0); - put("one", 1); - }}, - new Object2IntOpenHashMap<>() {{ - put("zero", 0); - put("one", 1); - put("two", 2); - }} + Object2IntMap[] map = new Object2IntOpenHashMap[]{new Object2IntOpenHashMap<>() { + { + put("zero", 0); + } + }, new Object2IntOpenHashMap<>() { + { + put("zero", 0); + put("one", 1); + } + }, new Object2IntOpenHashMap<>() { + { + put("zero", 0); + put("one", 1); + put("two", 2); + } + } }; SerializationTestUtil.forArrayType(new TypeReference>() { - }).objectMappers(getApiMapper(), getInternalCommunicationMapper(), getManagerInternalMapper()) - .customizingAssertion(RecursiveComparisonAssert::ignoringCollectionOrder) - .test(map); + }) + .objectMappers( + getApiMapper(), + getInternalCommunicationMapper(), + getManagerInternalMapper()) + .customizingAssertion(RecursiveComparisonAssert::ignoringCollectionOrder) + .test( + map); } @Test public void formBackendVersion() throws JSONException, IOException { - final FormBackendVersion version = new FormBackendVersion("3.45.45-g85ut85u43t8", ZonedDateTime.parse("2007-12-03T10:15:30+00:00")); + final FormBackendVersion version = new FormBackendVersion( + "3.45.45-g85ut85u43t8", + ZonedDateTime.parse("2007-12-03T10:15:30+00:00")); SerializationTestUtil.forType(FormBackendVersion.class) - .objectMappers(getApiMapper(), getManagerInternalMapper()) - .test(version); + .objectMappers( + getApiMapper(), + getManagerInternalMapper()) + .test(version); } } diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java b/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java index 7e9169a6444..56b8e383154 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java @@ -15,7 +15,7 @@ public class CopyUserTest { @Test - void testUserCopy(){ + void testUserCopy() { MetaStorage storage = new NonPersistentStoreFactory().createMetaStorage(); // Create test role @@ -38,7 +38,10 @@ void testUserCopy(){ User copy = AuthorizationController.flatCopyUser(originUser, "copytest", storage); // Check that it is not the same user - assertThat(copy).usingRecursiveComparison().ignoringFieldsOfTypes(User.ShiroUserAdapter.class).isNotEqualTo(originUser); + assertThat(copy).usingRecursiveComparison() + .ignoringFieldsOfTypes(User.ShiroUserAdapter.class) + .isNotEqualTo( + originUser); // Check that the copy does not have any mappings assertThat(group.containsUser(copy.getId())).isFalse(); diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/IdpDelegatingAccessTokenCreatorTest.java b/backend/src/test/java/com/bakdata/conquery/models/auth/IdpDelegatingAccessTokenCreatorTest.java index be42149d216..4127ce51e81 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/auth/IdpDelegatingAccessTokenCreatorTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/auth/IdpDelegatingAccessTokenCreatorTest.java @@ -9,8 +9,8 @@ import static org.mockserver.model.Parameter.param; import static org.mockserver.model.ParameterBody.params; -import java.util.Map; import jakarta.validation.Validator; +import java.util.Map; import com.auth0.jwt.JWT; import com.auth0.jwt.algorithms.Algorithm; @@ -32,7 +32,9 @@ @Slf4j public class IdpDelegatingAccessTokenCreatorTest { @RegisterExtension - private static final MockServerExtension OIDC_SERVER = new MockServerExtension(ClientAndServer.startClientAndServer(1080), IdpDelegatingAccessTokenCreatorTest::initOIDCServer); + private static final MockServerExtension OIDC_SERVER = new MockServerExtension( + ClientAndServer.startClientAndServer(1080), + IdpDelegatingAccessTokenCreatorTest::initOIDCServer); private static final IntrospectionDelegatingRealmFactory CONFIG = new IntrospectionDelegatingRealmFactory(); private static final Validator VALIDATOR = BaseValidator.newValidator(); @@ -40,7 +42,10 @@ public class IdpDelegatingAccessTokenCreatorTest { // User 1 private static final String USER_1_NAME = "test_name1"; private static final String USER_1_PASSWORD = "test_password1"; - private static final String USER_1_TOKEN = JWT.create().withClaim("name", USER_1_NAME).sign(Algorithm.HMAC256("secret")); + private static final String USER_1_TOKEN = JWT.create() + .withClaim("name", USER_1_NAME) + .sign( + Algorithm.HMAC256("secret")); private static IdpDelegatingAccessTokenCreator idpDelegatingAccessTokenCreator; @@ -67,29 +72,37 @@ private static void initOIDCServer(ClientAndServer clientAndServer) { OIDCMockServer.init(clientAndServer, (server) -> { // Mock username-password-for-token exchange server.when( - request().withMethod("POST").withPath(String.format("/realms/%s/protocol/openid-connect/token", OIDCMockServer.REALM_NAME)) - .withHeaders(header("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")).withBody( - params( - param("password", USER_1_PASSWORD), - param("grant_type", "password"), - param("username", USER_1_NAME), - param("scope", "openid")))) - .respond( - response() - .withBody(JsonBody.json( - new Object() { - @Getter - final String token_type = "Bearer"; - @Getter - final String access_token = USER_1_TOKEN; - } - ))); + request().withMethod("POST") + .withPath( + String.format("/realms/%s/protocol/openid-connect/token", OIDCMockServer.REALM_NAME)) + .withHeaders( + header("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")) + .withBody( + params( + param("password", USER_1_PASSWORD), + param("grant_type", "password"), + param("username", USER_1_NAME), + param("scope", "openid")))) + .respond( + response().withBody( + JsonBody.json( + new Object() { + @Getter + final String token_type = "Bearer"; + @Getter + final String access_token = USER_1_TOKEN; + } + ))); // Block other exchange requests (this has a lower prio than the above) server.when( - request().withMethod("POST").withPath(String.format("/realms/%s/protocol/openid-connect/token", OIDCMockServer.REALM_NAME))) - .respond( - response().withStatusCode(HttpStatus.SC_FORBIDDEN).withContentType(MediaType.APPLICATION_JSON_UTF_8) - .withBody("{\"error\" : \"Wrong username or password\"")); + request().withMethod("POST") + .withPath( + String.format("/realms/%s/protocol/openid-connect/token", OIDCMockServer.REALM_NAME))) + .respond( + response().withStatusCode(HttpStatus.SC_FORBIDDEN) + .withContentType( + MediaType.APPLICATION_JSON_UTF_8) + .withBody("{\"error\" : \"Wrong username or password\"")); }); } @@ -105,8 +118,8 @@ public void vaildUsernamePassword() { public void invaildUsernamePassword() { log.info("This test will print an Error below."); assertThatThrownBy( - () -> idpDelegatingAccessTokenCreator.createAccessToken(USER_1_NAME, "bad_password")) - .isInstanceOf(IllegalStateException.class); + () -> idpDelegatingAccessTokenCreator.createAccessToken(USER_1_NAME, "bad_password")).isInstanceOf( + IllegalStateException.class); } diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/InstancePermissionImplificationTest.java b/backend/src/test/java/com/bakdata/conquery/models/auth/InstancePermissionImplificationTest.java index 9a0b9bed9c7..b927b29d45d 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/auth/InstancePermissionImplificationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/auth/InstancePermissionImplificationTest.java @@ -16,68 +16,68 @@ import org.junit.jupiter.api.Test; public class InstancePermissionImplificationTest { - + private static final String DATASET1 = "dataset1"; private static final String DATASET2 = "dataset2"; - + @Test public void testEqual() { // Test equal Permissions Permission pStored = DatasetPermission.onInstance( - Ability.READ.asSet(), - new DatasetId(DATASET1)); + Ability.READ.asSet(), + new DatasetId(DATASET1)); Permission pRequested = DatasetPermission.onInstance( - Ability.READ.asSet(), - new DatasetId(DATASET1)); + Ability.READ.asSet(), + new DatasetId(DATASET1)); assert pStored.implies(pRequested); } - + @Test public void testDivergingPrincipals() { // Test different user principals Permission pStored = DatasetPermission.onInstance( - Ability.READ.asSet(), - new DatasetId(DATASET1)); + Ability.READ.asSet(), + new DatasetId(DATASET1)); Permission pRequested = DatasetPermission.onInstance( - Ability.READ.asSet(), - new DatasetId(DATASET1)); + Ability.READ.asSet(), + new DatasetId(DATASET1)); assert pStored.implies(pRequested); } - + @Test public void testDivergingAccesTypes() { // Test different access types Permission pStored = DatasetPermission.onInstance( - Ability.READ.asSet(), - new DatasetId(DATASET1)); + Ability.READ.asSet(), + new DatasetId(DATASET1)); Permission pRequested = DatasetPermission.onInstance( Ability.DOWNLOAD.asSet(), - new DatasetId(DATASET1)); + new DatasetId(DATASET1)); assert !pStored.implies(pRequested); } - + @Test public void testDivergingInstances() { // Test different Instances Permission pStored = DatasetPermission.onInstance( - Ability.READ.asSet(), - new DatasetId(DATASET1)); + Ability.READ.asSet(), + new DatasetId(DATASET1)); Permission pRequested = DatasetPermission.onInstance( - Ability.READ.asSet(), - new DatasetId(DATASET2)); + Ability.READ.asSet(), + new DatasetId(DATASET2)); assert !pStored.implies(pRequested); } - + @Test public void testMultipleAccessesProhibit() { // Test different Instances Permission pStored = DatasetPermission.onInstance( - Ability.READ.asSet(), - new DatasetId(DATASET1)); + Ability.READ.asSet(), + new DatasetId(DATASET1)); Permission pRequested = DatasetPermission.onInstance( EnumSet.of(Ability.READ, Ability.DOWNLOAD), - new DatasetId(DATASET1)); + new DatasetId(DATASET1)); // Should not imply, since one access is missing assert !pStored.implies(pRequested); } @@ -87,17 +87,19 @@ public void testMultipleAccessesPermit() { // Test different Instances Permission pStored = DatasetPermission.onInstance( EnumSet.of(Ability.READ, Ability.DOWNLOAD), - new DatasetId(DATASET1)); + new DatasetId(DATASET1)); Permission pRequested = DatasetPermission.onInstance( - Ability.READ.asSet(), - new DatasetId(DATASET1)); + Ability.READ.asSet(), + new DatasetId(DATASET1)); assert pStored.implies(pRequested); } - + @Test public void permissionTypesFail() { Permission dPerm = DatasetPermission.onInstance(Ability.READ.asSet(), new DatasetId(DATASET1)); - Permission qPerm = ExecutionPermission.onInstance(Ability.READ.asSet(), new ManagedExecutionId(new DatasetId(DATASET1), UUID.randomUUID())); + Permission qPerm = ExecutionPermission.onInstance( + Ability.READ.asSet(), + new ManagedExecutionId(new DatasetId(DATASET1), UUID.randomUUID())); Permission sPerm = SuperPermission.onDomain(); Permission aPerm = AdminPermission.onDomain(); assertThat(dPerm.implies(qPerm)).isFalse(); diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/IntrospectionDelegatingRealmTest.java b/backend/src/test/java/com/bakdata/conquery/models/auth/IntrospectionDelegatingRealmTest.java index 9a506186470..5f28f03cd66 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/auth/IntrospectionDelegatingRealmTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/auth/IntrospectionDelegatingRealmTest.java @@ -9,11 +9,11 @@ import static org.mockserver.model.Parameter.param; import static org.mockserver.model.ParameterBody.params; +import jakarta.validation.Validator; import java.net.URI; import java.util.Map; import java.util.Set; import java.util.UUID; -import jakarta.validation.Validator; import com.auth0.jwt.JWT; import com.auth0.jwt.algorithms.Algorithm; @@ -47,7 +47,9 @@ @Slf4j public class IntrospectionDelegatingRealmTest { @RegisterExtension - private static final MockServerExtension OIDC_SERVER = new MockServerExtension(ClientAndServer.startClientAndServer(), IntrospectionDelegatingRealmTest::initOIDCServer); + private static final MockServerExtension OIDC_SERVER = new MockServerExtension( + ClientAndServer.startClientAndServer(), + IntrospectionDelegatingRealmTest::initOIDCServer); @RegisterExtension @Order(0) @@ -66,10 +68,11 @@ public class IntrospectionDelegatingRealmTest { public static final String BACKEND_AUD = "backend"; public static final String SOME_SECRET = "secret"; private static final String USER_1_TOKEN = JWT.create() - .withSubject(USER_1_NAME) - .withAudience(BACKEND_AUD) - .withClaim("name", USER_1_NAME) - .sign(Algorithm.HMAC256(SOME_SECRET)); + .withSubject(USER_1_NAME) + .withAudience( + BACKEND_AUD) + .withClaim("name", USER_1_NAME) + .sign(Algorithm.HMAC256(SOME_SECRET)); private static final BearerToken USER1_TOKEN_WRAPPED = new BearerToken(USER_1_TOKEN); // User 2 @@ -78,10 +81,11 @@ public class IntrospectionDelegatingRealmTest { @RegisterExtension private static final UserExtension USER_2_EXTENSION = new UserExtension(STORAGE, USER_2_NAME, USER_2_LABEL); private static final String USER_2_TOKEN = JWT.create() - .withSubject(USER_2_NAME) - .withAudience(BACKEND_AUD) - .withClaim("name", USER_2_LABEL) - .sign(Algorithm.HMAC256(SOME_SECRET)); + .withSubject(USER_2_NAME) + .withAudience( + BACKEND_AUD) + .withClaim("name", USER_2_LABEL) + .sign(Algorithm.HMAC256(SOME_SECRET)); private static final BearerToken USER_2_TOKEN_WRAPPED = new BearerToken(USER_2_TOKEN); // User 3 existing @@ -90,10 +94,11 @@ public class IntrospectionDelegatingRealmTest { @RegisterExtension private static final UserExtension USER_3_EXTENSION = new UserExtension(STORAGE, USER_3_NAME, USER_3_LABEL); private static final String USER_3_TOKEN = JWT.create() - .withSubject(USER_3_NAME) - .withAudience(BACKEND_AUD) - .withClaim("name", USER_3_LABEL) - .sign(Algorithm.HMAC256(SOME_SECRET)); + .withSubject(USER_3_NAME) + .withAudience( + BACKEND_AUD) + .withClaim("name", USER_3_LABEL) + .sign(Algorithm.HMAC256(SOME_SECRET)); private static final BearerToken USER_3_TOKEN_WRAPPED = new BearerToken(USER_3_TOKEN); // Groups @@ -104,7 +109,8 @@ public class IntrospectionDelegatingRealmTest { private static final String GROUPNAME_2 = "group2"; // Group is created during test public static KeycloakGroup KEYCLOAK_GROUP_2; - public static final URI FRONT_CHANNEL_LOGOUT = URI.create("http://localhost:%d/realms/test_realm/protocol/openid-connect/logout".formatted(OIDC_SERVER.getPort())); + public static final URI FRONT_CHANNEL_LOGOUT = URI.create( + "http://localhost:%d/realms/test_realm/protocol/openid-connect/logout".formatted(OIDC_SERVER.getPort())); private static TestRealm REALM; @@ -112,20 +118,29 @@ public class IntrospectionDelegatingRealmTest { @BeforeAll public static void beforeAll() { - KEYCLOAK_GROUP_1 = new KeycloakGroup(UUID.randomUUID().toString(), "Group1", "g1", Map.of(GROUP_ID_ATTRIBUTE, GROUP_1_EXISTING_EXTENSION.getGroup().getId().toString()), Set.of()); - KEYCLOAK_GROUP_2 = new KeycloakGroup(UUID.randomUUID().toString(), "Group2", "g2", Map.of(GROUP_ID_ATTRIBUTE, new GroupId(GROUPNAME_2).toString()), Set.of()); + KEYCLOAK_GROUP_1 = new KeycloakGroup( + UUID.randomUUID().toString(), + "Group1", + "g1", + Map.of(GROUP_ID_ATTRIBUTE, GROUP_1_EXISTING_EXTENSION.getGroup().getId().toString()), + Set.of()); + KEYCLOAK_GROUP_2 = new KeycloakGroup( + UUID.randomUUID().toString(), + "Group2", + "g2", + Map.of(GROUP_ID_ATTRIBUTE, new GroupId(GROUPNAME_2).toString()), + Set.of()); KEYCLOAK_API = mock(KeycloakApi.class); - doAnswer(invocation -> Set.of(KEYCLOAK_GROUP_1, KEYCLOAK_GROUP_2)).when(KEYCLOAK_API) - .getGroupHierarchy(); + doAnswer(invocation -> Set.of(KEYCLOAK_GROUP_1, KEYCLOAK_GROUP_2)).when(KEYCLOAK_API).getGroupHierarchy(); doAnswer( - invocation -> { - final String userId = invocation.getArgument(0); - if (userId.equals(USER_2_NAME)) { - return Set.of(KEYCLOAK_GROUP_1, KEYCLOAK_GROUP_2); - } - return Set.of(); + invocation -> { + final String userId = invocation.getArgument(0); + if (userId.equals(USER_2_NAME)) { + return Set.of(KEYCLOAK_GROUP_1, KEYCLOAK_GROUP_2); } + return Set.of(); + } ).when(KEYCLOAK_API).getUserGroups(any(String.class)); initRealm(); @@ -160,57 +175,93 @@ private static void initRealm() { private static void initOIDCServer(ClientAndServer mockServer) { // Mock username-password-for-token exchange OIDCMockServer.init( - mockServer, - (server) -> { - server.when( - request().withMethod("POST").withPath(String.format("/realms/%s/protocol/openid-connect/token", OIDCMockServer.REALM_NAME)) - .withHeaders(header("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")).withBody( - params( - param("password", USER_1_PASSWORD), - param("grant_type", "password"), - param("username", USER_1_NAME), - param("scope", "openid") - ))) - .respond( - response().withContentType(MediaType.APPLICATION_JSON_UTF_8) - .withBody("{\"token_type\" : \"Bearer\",\"access_token\" : \"" + USER_1_TOKEN + "\"}")); - - // Block other exchange requests (this has a lower prio than the above) - server.when( - request().withMethod("POST").withPath(String.format("/realms/%s/protocol/openid-connect/token", OIDCMockServer.REALM_NAME))) - .respond( - response().withStatusCode(HttpStatus.SC_FORBIDDEN).withContentType(MediaType.APPLICATION_JSON_UTF_8) - .withBody("{\"error\" : \"Wrong username or password\"")); - - // Mock token introspection - // For USER 1 - server.when( - request().withMethod("POST") - .withPath(String.format("/realms/%s/protocol/openid-connect/token/introspect", OIDCMockServer.REALM_NAME)) - .withHeaders(header("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")) - .withBody(params(param("token_type_hint", "access_token"), param("token", USER_1_TOKEN)))) - .respond( - response().withContentType(MediaType.APPLICATION_JSON_UTF_8) - .withBody("{\"username\" : \"" + USER_1_NAME + "\", \"active\": true}")); - // For USER 2 - server.when( - request().withMethod("POST") - .withPath(String.format("/realms/%s/protocol/openid-connect/token/introspect", OIDCMockServer.REALM_NAME)) - .withHeaders(header("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")) - .withBody(params(param("token_type_hint", "access_token"), param("token", USER_2_TOKEN)))) - .respond( - response().withContentType(MediaType.APPLICATION_JSON_UTF_8) - .withBody("{\"username\" : \"" + USER_2_NAME + "\",\"name\" : \"" + USER_2_LABEL + "\", \"active\": true}")); - // For USER 3 - server.when( - request().withMethod("POST") - .withPath(String.format("/realms/%s/protocol/openid-connect/token/introspect", OIDCMockServer.REALM_NAME)) - .withHeaders(header("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")) - .withBody(params(param("token_type_hint", "access_token"), param("token", USER_3_TOKEN)))) - .respond( - response().withContentType(MediaType.APPLICATION_JSON_UTF_8) - .withBody("{\"username\" : \"" + USER_3_NAME + "\",\"name\" : \"" + USER_3_LABEL + "\", \"active\": true}")); - } + mockServer, + (server) -> { + server.when( + request().withMethod("POST") + .withPath( + String.format( + "/realms/%s/protocol/openid-connect/token", + OIDCMockServer.REALM_NAME)) + .withHeaders( + header("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")) + .withBody( + params( + param("password", USER_1_PASSWORD), + param("grant_type", "password"), + param("username", USER_1_NAME), + param("scope", "openid") + ))) + .respond( + response().withContentType(MediaType.APPLICATION_JSON_UTF_8) + .withBody( + "{\"token_type\" : \"Bearer\",\"access_token\" : \"" + USER_1_TOKEN + "\"}")); + + // Block other exchange requests (this has a lower prio than the above) + server.when( + request().withMethod("POST") + .withPath( + String.format("/realms/%s/protocol/openid-connect/token", OIDCMockServer.REALM_NAME))) + .respond( + response().withStatusCode(HttpStatus.SC_FORBIDDEN) + .withContentType( + MediaType.APPLICATION_JSON_UTF_8) + .withBody( + "{\"error\" : \"Wrong username or password\"")); + + // Mock token introspection + // For USER 1 + server.when( + request().withMethod("POST") + .withPath( + String.format( + "/realms/%s/protocol/openid-connect/token/introspect", + OIDCMockServer.REALM_NAME)) + .withHeaders( + header("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")) + .withBody( + params( + param("token_type_hint", "access_token"), + param("token", USER_1_TOKEN)))) + .respond( + response().withContentType(MediaType.APPLICATION_JSON_UTF_8) + .withBody( + "{\"username\" : \"" + USER_1_NAME + "\", \"active\": true}")); + // For USER 2 + server.when( + request().withMethod("POST") + .withPath( + String.format( + "/realms/%s/protocol/openid-connect/token/introspect", + OIDCMockServer.REALM_NAME)) + .withHeaders( + header("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")) + .withBody( + params( + param("token_type_hint", "access_token"), + param("token", USER_2_TOKEN)))) + .respond( + response().withContentType(MediaType.APPLICATION_JSON_UTF_8) + .withBody( + "{\"username\" : \"" + USER_2_NAME + "\",\"name\" : \"" + USER_2_LABEL + "\", \"active\": true}")); + // For USER 3 + server.when( + request().withMethod("POST") + .withPath( + String.format( + "/realms/%s/protocol/openid-connect/token/introspect", + OIDCMockServer.REALM_NAME)) + .withHeaders( + header("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")) + .withBody( + params( + param("token_type_hint", "access_token"), + param("token", USER_3_TOKEN)))) + .respond( + response().withContentType(MediaType.APPLICATION_JSON_UTF_8) + .withBody( + "{\"username\" : \"" + USER_3_NAME + "\",\"name\" : \"" + USER_3_LABEL + "\", \"active\": true}")); + } ); } @@ -218,13 +269,20 @@ private static void initOIDCServer(ClientAndServer mockServer) { public void tokenIntrospectionSimpleUserNew() { AuthenticationInfo info = REALM.doGetAuthenticationInfo(USER1_TOKEN_WRAPPED); - assertThat(info) - .usingRecursiveComparison() - .usingOverriddenEquals() - .ignoringFields(ConqueryAuthenticationInfo.Fields.credentials) - .ignoringFieldsOfTypes(User.ShiroUserAdapter.class) - .isEqualTo(new ConqueryAuthenticationInfo(USER_1_EXTENSION.getUser(), USER1_TOKEN_WRAPPED, REALM, true, FRONT_CHANNEL_LOGOUT)); - assertThat(STORAGE.getAllUsers()).containsOnly(new User(USER_1_NAME, USER_1_NAME, STORAGE_EXTENTION.getMetaStorage())); + assertThat(info).usingRecursiveComparison() + .usingOverriddenEquals() + .ignoringFields( + ConqueryAuthenticationInfo.Fields.credentials) + .ignoringFieldsOfTypes(User.ShiroUserAdapter.class) + .isEqualTo( + new ConqueryAuthenticationInfo( + USER_1_EXTENSION.getUser(), + USER1_TOKEN_WRAPPED, + REALM, + true, + FRONT_CHANNEL_LOGOUT)); + assertThat(STORAGE.getAllUsers()).containsOnly( + new User(USER_1_NAME, USER_1_NAME, STORAGE_EXTENTION.getMetaStorage())); } @Test @@ -232,11 +290,17 @@ public void tokenIntrospectionSimpleUserExisting() { AuthenticationInfo info = REALM.doGetAuthenticationInfo(USER1_TOKEN_WRAPPED); - assertThat(info) - .usingRecursiveComparison() - .usingOverriddenEquals() - .ignoringFields(ConqueryAuthenticationInfo.Fields.credentials) - .isEqualTo(new ConqueryAuthenticationInfo(USER_1_EXTENSION.getUser(), USER1_TOKEN_WRAPPED, REALM, true, FRONT_CHANNEL_LOGOUT)); + assertThat(info).usingRecursiveComparison() + .usingOverriddenEquals() + .ignoringFields( + ConqueryAuthenticationInfo.Fields.credentials) + .isEqualTo( + new ConqueryAuthenticationInfo( + USER_1_EXTENSION.getUser(), + USER1_TOKEN_WRAPPED, + REALM, + true, + FRONT_CHANNEL_LOGOUT)); assertThat(STORAGE.getAllUsers()).containsOnly(USER_1_EXTENSION.getUser()); } @@ -245,11 +309,13 @@ public void tokenIntrospectionGroupedUser() { AuthenticationInfo info = REALM.doGetAuthenticationInfo(USER_2_TOKEN_WRAPPED); - final ConqueryAuthenticationInfo expected = new ConqueryAuthenticationInfo(USER_2_EXTENSION.getUser(), USER_2_TOKEN_WRAPPED, REALM, true, FRONT_CHANNEL_LOGOUT); - assertThat(info) - .usingRecursiveComparison() - .usingOverriddenEquals() - .isEqualTo(expected); + final ConqueryAuthenticationInfo expected = new ConqueryAuthenticationInfo( + USER_2_EXTENSION.getUser(), + USER_2_TOKEN_WRAPPED, + REALM, + true, + FRONT_CHANNEL_LOGOUT); + assertThat(info).usingRecursiveComparison().usingOverriddenEquals().isEqualTo(expected); assertThat(STORAGE.getAllUsers()).containsOnly(USER_2_EXTENSION.getUser()); assertThat(STORAGE.getAllGroups()).hasSize(2); // Pre-existing group and a second group that has been added in the process assertThat(STORAGE.getGroup(new GroupId(GROUPNAME_1)).getMembers()).contains(new UserId(USER_2_NAME)); @@ -264,13 +330,19 @@ public void tokenIntrospectionGroupedUserRemoveGroupMapping() { AuthenticationInfo info = REALM.doGetAuthenticationInfo(USER_3_TOKEN_WRAPPED); - assertThat(info) - .usingRecursiveComparison() - .usingOverriddenEquals() - .ignoringFields(ConqueryAuthenticationInfo.Fields.credentials) - .isEqualTo(new ConqueryAuthenticationInfo(USER_3_EXTENSION.getUser(), USER_3_TOKEN_WRAPPED, REALM, true, FRONT_CHANNEL_LOGOUT)); + assertThat(info).usingRecursiveComparison() + .usingOverriddenEquals() + .ignoringFields( + ConqueryAuthenticationInfo.Fields.credentials) + .isEqualTo( + new ConqueryAuthenticationInfo( + USER_3_EXTENSION.getUser(), + USER_3_TOKEN_WRAPPED, + REALM, + true, + FRONT_CHANNEL_LOGOUT)); assertThat(STORAGE.getAllUsers()).containsOnly(USER_3_EXTENSION.getUser()); - assertThat(STORAGE.getAllGroups()).hasSize(1); // Pre-existing group + assertThat(STORAGE.getAllGroups()).hasSize(1); // Pre-existing group assertThat(STORAGE.getGroup(new GroupId(GROUPNAME_1)).getMembers()).doesNotContain(new UserId(USER_3_NAME)); } diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/LocalAuthRealmTest.java b/backend/src/test/java/com/bakdata/conquery/models/auth/LocalAuthRealmTest.java index af1e0d78fd8..0f42817a983 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/auth/LocalAuthRealmTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/auth/LocalAuthRealmTest.java @@ -39,7 +39,7 @@ @TestInstance(Lifecycle.PER_CLASS) public class LocalAuthRealmTest { - private final MetaStorage storage = new NonPersistentStoreFactory().createMetaStorage(); + private final MetaStorage storage = new NonPersistentStoreFactory().createMetaStorage(); private File tmpDir; private LocalAuthenticationRealm realm; private User user1; @@ -53,18 +53,18 @@ public void setupAll() throws Exception { conqueryTokenRealm = new ConqueryTokenRealm(storage); - realm = - new LocalAuthenticationRealm( - Validators.newValidator(), - Jackson.BINARY_MAPPER, conqueryTokenRealm, - "localTestRealm", - tmpDir, - new XodusConfig(), - Duration.hours(4), - BcryptFunction.getInstance(4), - CaffeineSpec.parse(""), - null // no metrics - ); // 4 is minimum + realm = new LocalAuthenticationRealm( + Validators.newValidator(), + Jackson.BINARY_MAPPER, + conqueryTokenRealm, + "localTestRealm", + tmpDir, + new XodusConfig(), + Duration.hours(4), + BcryptFunction.getInstance(4), + CaffeineSpec.parse(""), + null // no metrics + ); // 4 is minimum LifecycleUtils.init(realm); } @@ -92,26 +92,26 @@ public void cleanUpAll() { @Test public void testEmptyUsername() { - assertThatThrownBy(() -> realm.createAccessToken("", "testPassword")) - .isInstanceOf(IncorrectCredentialsException.class).hasMessageContaining("Username was empty"); + assertThatThrownBy(() -> realm.createAccessToken("", "testPassword")).isInstanceOf( + IncorrectCredentialsException.class).hasMessageContaining("Username was empty"); } @Test public void testEmptyPassword() { - assertThatThrownBy(() -> realm.createAccessToken("TestUser", "")) - .isInstanceOf(IncorrectCredentialsException.class).hasMessageContaining("Password was empty"); + assertThatThrownBy(() -> realm.createAccessToken("TestUser", "")).isInstanceOf( + IncorrectCredentialsException.class).hasMessageContaining("Password was empty"); } @Test public void testWrongPassword() { - assertThatThrownBy(() -> realm.createAccessToken("TestUser", "wrongPassword")) - .isInstanceOf(IncorrectCredentialsException.class).hasMessageContaining("Password was was invalid for user"); + assertThatThrownBy(() -> realm.createAccessToken("TestUser", "wrongPassword")).isInstanceOf( + IncorrectCredentialsException.class).hasMessageContaining("Password was was invalid for user"); } @Test public void testWrongUsername() { - assertThatThrownBy(() -> realm.createAccessToken("NoTestUser", "testPassword")) - .isInstanceOf(CredentialsException.class).hasMessageContaining("No password hash was found for user"); + assertThatThrownBy(() -> realm.createAccessToken("NoTestUser", "testPassword")).isInstanceOf( + CredentialsException.class).hasMessageContaining("No password hash was found for user"); } @Test @@ -120,8 +120,9 @@ public void testValidUsernamePassword() { String jwt = realm.createAccessToken("TestUser", "testPassword"); assertThatCode(() -> JWT.decode(jwt)).doesNotThrowAnyException(); - assertThat(conqueryTokenRealm.doGetAuthenticationInfo(new BearerToken(jwt)).getPrincipals().getPrimaryPrincipal()) - .isEqualTo(user1); + assertThat( + conqueryTokenRealm.doGetAuthenticationInfo( + new BearerToken(jwt)).getPrincipals().getPrimaryPrincipal()).isEqualTo(user1); } @Test @@ -129,8 +130,8 @@ public void testUserUpdate() { realm.updateUser(user1.getId(), new PasswordCredential("newTestPassword")); // Wrong (old) password - assertThatThrownBy(() -> realm.createAccessToken("TestUser", "testPassword")) - .isInstanceOf(IncorrectCredentialsException.class).hasMessageContaining("Password was was invalid for user"); + assertThatThrownBy(() -> realm.createAccessToken("TestUser", "testPassword")).isInstanceOf( + IncorrectCredentialsException.class).hasMessageContaining("Password was was invalid for user"); // Right (new) password String jwt = realm.createAccessToken("TestUser", "newTestPassword"); @@ -141,8 +142,8 @@ public void testUserUpdate() { public void testRemoveUser() { realm.removeUser(user1.getId()); // Wrong password - assertThatThrownBy(() -> realm.createAccessToken("TestUser", "testPassword")) - .isInstanceOf(CredentialsException.class).hasMessageContaining("No password hash was found for user"); + assertThatThrownBy(() -> realm.createAccessToken("TestUser", "testPassword")).isInstanceOf( + CredentialsException.class).hasMessageContaining("No password hash was found for user"); } } diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/OIDCMockServer.java b/backend/src/test/java/com/bakdata/conquery/models/auth/OIDCMockServer.java index 7d46813ad2f..3faa23d37a5 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/auth/OIDCMockServer.java +++ b/backend/src/test/java/com/bakdata/conquery/models/auth/OIDCMockServer.java @@ -22,53 +22,55 @@ public class OIDCMockServer { public static final String REALM_NAME = "test_realm"; public static void init(ClientAndServer server) { - init(server, (_server) -> {}); + init(server, (_server) -> { + }); } public static void init(ClientAndServer server, Consumer testMappings) { String mockServerUrl = "http://localhost:%d".formatted(server.getPort()); // Mock well-known discovery endpoint (this is actually the output of keycloak) - server.when(request().withMethod("GET").withPath("/realms/" + REALM_NAME + "/.well-known/uma2-configuration")) - .respond( - response().withBody( - JsonBody.json( - new Object() { - @Getter - final String issuer = mockServerUrl; - @Getter - final String authorization_endpoint = mockServerUrl + "/realms/" + REALM_NAME + "/protocol/openid-connect/auth"; - @Getter - final String token_endpoint = mockServerUrl + "/realms/" + REALM_NAME + "/protocol/openid-connect/token"; - @Getter - final String introspection_endpoint = mockServerUrl + "/realms/" + REALM_NAME + "/protocol/openid-connect/token/introspect"; - @Getter - final String end_session_endpoint = mockServerUrl + "/realms/" + REALM_NAME + "/protocol/openid-connect/logout"; - @Getter - final String jwks_uri = mockServerUrl + "/realms/" + REALM_NAME + "/protocol/openid-connect/certs"; - @Getter - final String[] grant_types_supported = {"authorization_code", "implicit", "refresh_token", "password", "client_credentials"}; - @Getter - final String[] response_types_supported = {"code", "none", "id_token", "token", "id_token token", "code id_token", "code token", "code id_token token"}; - @Getter - final String[] response_modes_supported = {"query", "fragment", "form_post"}; - @Getter - final String registration_endpoint = mockServerUrl + "/realms/" + REALM_NAME + "/clients-registrations/openid-connect"; - @Getter - final String[] token_endpoint_auth_methods_supported = {"private_key_jwt", "client_secret_basic", "client_secret_post", "tls_client_auth", "client_secret_jwt"}; - @Getter - final String[] token_endpoint_auth_signing_alg_values_supported = {"PS384", "ES384", "RS384", "HS256", "HS512", "ES256", "RS256", "HS384", "ES512", "PS256", "PS512", "RS512"}; - @Getter - final String[] scopes_supported = {"openid", "address", "email", "microprofile-jwt", "offline_access", "phone", "profile", "roles", "web-origins"}; - @Getter - final String resource_registration_endpoint = mockServerUrl + "/realms/" + REALM_NAME + "/authz/protection/resource_set"; - @Getter - final String permission_endpoint = mockServerUrl + "/realms/" + REALM_NAME + "/authz/protection/permission"; - @Getter - final String policy_endpoint = mockServerUrl + "/realms/" + REALM_NAME + "/authz/protection/uma-policy"; - } - ) - )); + server.when( + request().withMethod("GET").withPath("/realms/" + REALM_NAME + "/.well-known/uma2-configuration")) + .respond( + response().withBody( + JsonBody.json( + new Object() { + @Getter + final String issuer = mockServerUrl; + @Getter + final String authorization_endpoint = mockServerUrl + "/realms/" + REALM_NAME + "/protocol/openid-connect/auth"; + @Getter + final String token_endpoint = mockServerUrl + "/realms/" + REALM_NAME + "/protocol/openid-connect/token"; + @Getter + final String introspection_endpoint = mockServerUrl + "/realms/" + REALM_NAME + "/protocol/openid-connect/token/introspect"; + @Getter + final String end_session_endpoint = mockServerUrl + "/realms/" + REALM_NAME + "/protocol/openid-connect/logout"; + @Getter + final String jwks_uri = mockServerUrl + "/realms/" + REALM_NAME + "/protocol/openid-connect/certs"; + @Getter + final String[] grant_types_supported = {"authorization_code", "implicit", "refresh_token", "password", "client_credentials"}; + @Getter + final String[] response_types_supported = {"code", "none", "id_token", "token", "id_token token", "code id_token", "code token", "code id_token token"}; + @Getter + final String[] response_modes_supported = {"query", "fragment", "form_post"}; + @Getter + final String registration_endpoint = mockServerUrl + "/realms/" + REALM_NAME + "/clients-registrations/openid-connect"; + @Getter + final String[] token_endpoint_auth_methods_supported = {"private_key_jwt", "client_secret_basic", "client_secret_post", "tls_client_auth", "client_secret_jwt"}; + @Getter + final String[] token_endpoint_auth_signing_alg_values_supported = {"PS384", "ES384", "RS384", "HS256", "HS512", "ES256", "RS256", "HS384", "ES512", "PS256", "PS512", "RS512"}; + @Getter + final String[] scopes_supported = {"openid", "address", "email", "microprofile-jwt", "offline_access", "phone", "profile", "roles", "web-origins"}; + @Getter + final String resource_registration_endpoint = mockServerUrl + "/realms/" + REALM_NAME + "/authz/protection/resource_set"; + @Getter + final String permission_endpoint = mockServerUrl + "/realms/" + REALM_NAME + "/authz/protection/permission"; + @Getter + final String policy_endpoint = mockServerUrl + "/realms/" + REALM_NAME + "/authz/protection/uma-policy"; + } + ) + )); // Register test provided mappings testMappings.accept(server); @@ -79,11 +81,11 @@ public static void init(ClientAndServer server, Consumer testMa @Override public HttpResponse handle(HttpRequest httpRequest) throws Exception { log.error( - "{} on {}\n\t Headers: {}\n\tBody {}", - httpRequest.getMethod(), - httpRequest.getPath(), - httpRequest.getHeaderList(), - httpRequest.getBodyAsString() + "{} on {}\n\t Headers: {}\n\tBody {}", + httpRequest.getMethod(), + httpRequest.getPath(), + httpRequest.getHeaderList(), + httpRequest.getBodyAsString() ); fail("Trapped because request did not match. See log."); return null; diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/PermissionCreationTest.java b/backend/src/test/java/com/bakdata/conquery/models/auth/PermissionCreationTest.java index feb73ff538d..0e1528cfe7d 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/auth/PermissionCreationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/auth/PermissionCreationTest.java @@ -2,26 +2,26 @@ import static org.assertj.core.api.Assertions.assertThat; -import org.apache.shiro.authz.Permission; -import org.junit.jupiter.api.Test; - import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.auth.permissions.DatasetPermission; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; +import org.apache.shiro.authz.Permission; +import org.junit.jupiter.api.Test; public class PermissionCreationTest { @Test public void createPermissionLegalAbility() { - assertThat(DatasetPermission.onInstance(Ability.READ.asSet(), new DatasetId("test"))).isInstanceOf(Permission.class); + assertThat(DatasetPermission.onInstance(Ability.READ.asSet(), new DatasetId("test"))).isInstanceOf( + Permission.class); } - + @Test public void createPermissionIllegalAbility() { - Permission perm = null ; + Permission perm = null; try { // This should fail because the ability is not allowed for a DatasetPermission perm = DatasetPermission.onInstance(Ability.SHARE.asSet(), new DatasetId("test")); - }catch (Exception e) { + } catch (Exception e) { assertThat(e).isInstanceOf(IllegalArgumentException.class); } // Should not be reached diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/GroupUtilTest.java b/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/GroupUtilTest.java index b38d4c132a8..9d696b2b6e3 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/GroupUtilTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/GroupUtilTest.java @@ -15,72 +15,118 @@ public class GroupUtilTest { private static final Set HIERARCHY = Set.of( - new KeycloakGroup("a", "A", "a", Map.of(), Set.of( - new KeycloakGroup("aa", "A", "a/a", Map.of(), Set.of()), - new KeycloakGroup("ab", "B", "a/b", Map.of(), Set.of()), - new KeycloakGroup("ac", "C", "a/c", Map.of(), Set.of()) + new KeycloakGroup( + "a", + "A", + "a", + Map.of(), + Set.of( + new KeycloakGroup("aa", "A", "a/a", Map.of(), Set.of()), + new KeycloakGroup("ab", "B", "a/b", Map.of(), Set.of()), + new KeycloakGroup("ac", "C", "a/c", Map.of(), Set.of()) )), - new KeycloakGroup("b", "B", "b", Map.of(), Set.of( - new KeycloakGroup("ba", "A", "b/a", Map.of(), Set.of()), - new KeycloakGroup("bb", "B", "b/b", Map.of(), Set.of( + new KeycloakGroup( + "b", + "B", + "b", + Map.of(), + Set.of( + new KeycloakGroup("ba", "A", "b/a", Map.of(), Set.of()), + new KeycloakGroup( + "bb", + "B", + "b/b", + Map.of(), + Set.of( - new KeycloakGroup("bba", "A", "b/b/a", Map.of(), Set.of()), - new KeycloakGroup("bbb", "B", "b/b/b", Map.of(), Set.of()), - new KeycloakGroup("bbc", "C", "b/b/c", Map.of(), Set.of()) + new KeycloakGroup("bba", "A", "b/b/a", Map.of(), Set.of()), + new KeycloakGroup("bbb", "B", "b/b/b", Map.of(), Set.of()), + new KeycloakGroup("bbc", "C", "b/b/c", Map.of(), Set.of()) )), - new KeycloakGroup("bc", "C", "b/c", Map.of(), Set.of()) + new KeycloakGroup("bc", "C", "b/c", Map.of(), Set.of()) )), - new KeycloakGroup("c", "C", "c", Map.of(), Set.of()) + new KeycloakGroup("c", "C", "c", Map.of(), Set.of()) ); @Test public void findTopLevel() { - final Collection actual = GroupUtil.getParentGroups(new KeycloakGroup("c", "C", "c", Map.of(), Set.of()), HIERARCHY); + final Collection actual = GroupUtil.getParentGroups( + new KeycloakGroup("c", "C", "c", Map.of(), Set.of()), + HIERARCHY); assertThat(actual).containsExactly(new KeycloakGroup("c", "C", "c", Map.of(), Set.of())); } @Test public void findMidLevel() { - final Collection actual = GroupUtil.getParentGroups(new KeycloakGroup("ba", "A", "b/a", Map.of(), Set.of()), HIERARCHY); + final Collection actual = GroupUtil.getParentGroups( + new KeycloakGroup("ba", "A", "b/a", Map.of(), Set.of()), + HIERARCHY); assertThat(actual).containsExactly( - new KeycloakGroup("ba", "A", "b/a", Map.of(), Set.of()), - new KeycloakGroup("b", "B", "b", Map.of(), Set.of( - new KeycloakGroup("ba", "A", "b/a", Map.of(), Set.of()), - new KeycloakGroup("bb", "B", "b/b", Map.of(), Set.of( + new KeycloakGroup("ba", "A", "b/a", Map.of(), Set.of()), + new KeycloakGroup( + "b", + "B", + "b", + Map.of(), + Set.of( + new KeycloakGroup("ba", "A", "b/a", Map.of(), Set.of()), + new KeycloakGroup( + "bb", + "B", + "b/b", + Map.of(), + Set.of( - new KeycloakGroup("bba", "A", "b/b/a", Map.of(), Set.of()), - new KeycloakGroup("bbb", "B", "b/b/b", Map.of(), Set.of()), - new KeycloakGroup("bbc", "C", "b/b/c", Map.of(), Set.of()) + new KeycloakGroup("bba", "A", "b/b/a", Map.of(), Set.of()), + new KeycloakGroup("bbb", "B", "b/b/b", Map.of(), Set.of()), + new KeycloakGroup("bbc", "C", "b/b/c", Map.of(), Set.of()) )), - new KeycloakGroup("bc", "C", "b/c", Map.of(), Set.of()) + new KeycloakGroup("bc", "C", "b/c", Map.of(), Set.of()) )) ); } @Test public void findLowLevel() { - final Collection actual = GroupUtil.getParentGroups(new KeycloakGroup("bba", "A", "b/b/a", Map.of(), Set.of()), HIERARCHY); + final Collection actual = GroupUtil.getParentGroups( + new KeycloakGroup("bba", "A", "b/b/a", Map.of(), Set.of()), + HIERARCHY); assertThat(actual).containsExactly( - new KeycloakGroup("bba", "A", "b/b/a", Map.of(), Set.of()), - new KeycloakGroup("bb", "B", "b/b", Map.of(), Set.of( - - new KeycloakGroup("bba", "A", "b/b/a", Map.of(), Set.of()), - new KeycloakGroup("bbb", "B", "b/b/b", Map.of(), Set.of()), - new KeycloakGroup("bbc", "C", "b/b/c", Map.of(), Set.of()) + new KeycloakGroup("bba", "A", "b/b/a", Map.of(), Set.of()), + new KeycloakGroup( + "bb", + "B", + "b/b", + Map.of(), + Set.of( + + new KeycloakGroup("bba", "A", "b/b/a", Map.of(), Set.of()), + new KeycloakGroup("bbb", "B", "b/b/b", Map.of(), Set.of()), + new KeycloakGroup("bbc", "C", "b/b/c", Map.of(), Set.of()) )), - new KeycloakGroup("b", "B", "b", Map.of(), Set.of( - new KeycloakGroup("ba", "A", "b/a", Map.of(), Set.of()), - new KeycloakGroup("bb", "B", "b/b", Map.of(), Set.of( + new KeycloakGroup( + "b", + "B", + "b", + Map.of(), + Set.of( + new KeycloakGroup("ba", "A", "b/a", Map.of(), Set.of()), + new KeycloakGroup( + "bb", + "B", + "b/b", + Map.of(), + Set.of( - new KeycloakGroup("bba", "A", "b/b/a", Map.of(), Set.of()), - new KeycloakGroup("bbb", "B", "b/b/b", Map.of(), Set.of()), - new KeycloakGroup("bbc", "C", "b/b/c", Map.of(), Set.of()) + new KeycloakGroup("bba", "A", "b/b/a", Map.of(), Set.of()), + new KeycloakGroup("bbb", "B", "b/b/b", Map.of(), Set.of()), + new KeycloakGroup("bbc", "C", "b/b/c", Map.of(), Set.of()) )), - new KeycloakGroup("bc", "C", "b/c", Map.of(), Set.of()) + new KeycloakGroup("bc", "C", "b/c", Map.of(), Set.of()) )) ); } @@ -88,14 +134,14 @@ public void findLowLevel() { @Test public void throwOnUnknownLeafGroup() { assertThatCode( - () -> GroupUtil.getParentGroups(new KeycloakGroup("baa", "A", "b/a/a", Map.of(), Set.of()), HIERARCHY) + () -> GroupUtil.getParentGroups(new KeycloakGroup("baa", "A", "b/a/a", Map.of(), Set.of()), HIERARCHY) ).isInstanceOf(NoSuchElementException.class); } @Test public void throwOnUnknownSiblingGroup() { assertThatCode( - () -> GroupUtil.getParentGroups(new KeycloakGroup("bd", "D", "b/d", Map.of(), Set.of()), HIERARCHY) + () -> GroupUtil.getParentGroups(new KeycloakGroup("bd", "D", "b/d", Map.of(), Set.of()), HIERARCHY) ).isInstanceOf(NoSuchElementException.class); } diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/IdpConfigRetrievalTest.java b/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/IdpConfigRetrievalTest.java index cf95a1e09b9..1b35a5ebeac 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/IdpConfigRetrievalTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/IdpConfigRetrievalTest.java @@ -4,8 +4,8 @@ import static org.mockserver.model.HttpRequest.request; import static org.mockserver.model.HttpResponse.response; -import java.net.URI; import jakarta.ws.rs.client.Client; +import java.net.URI; import com.bakdata.conquery.models.auth.OIDCMockServer; import com.bakdata.conquery.models.config.auth.JwtPkceVerifyingRealmFactory; @@ -19,25 +19,33 @@ public class IdpConfigRetrievalTest { @RegisterExtension - private static final MockServerExtension OIDC_SERVER = new MockServerExtension(ClientAndServer.startClientAndServer(), IdpConfigRetrievalTest::init); + private static final MockServerExtension OIDC_SERVER = new MockServerExtension( + ClientAndServer.startClientAndServer(), + IdpConfigRetrievalTest::init); private static final JwtPkceVerifyingRealmFactory REALM_FACTORY = new JwtPkceVerifyingRealmFactory(); - private static final Client CLIENT = new JerseyClientBuilder(new Environment("oidc-test")).build("oidc-test-client"); + private static final Client CLIENT = new JerseyClientBuilder(new Environment("oidc-test")).build( + "oidc-test-client"); private static void init(ClientAndServer mockServer) { OIDCMockServer.init(mockServer, (server) -> { // MOCK JWK Endpoint (1 signing + 1 encryption key) - server.when(request().withMethod("GET").withPath("/realms/" + OIDCMockServer.REALM_NAME + "/protocol/openid-connect/certs")) - .respond( - response().withBody( - JsonBody.json("{\"keys\":[{\"kid\":\"nW8YlwzTMqqOd6eqL1IXMZfBL7k2rSl8fX_NR80btvM\",\"kty\":\"RSA\",\"alg\":\"RS256\",\"use\":\"sig\",\"n\":\"uUdwWKsu_ExHC7-VxS84ZbOxJBUK_DmMYvfbvmufPX5Vv1Rhg2DMB1oy1hrz2cJVA7cbnKpXuHj5DC6BJ2RuO-zaC9bXBmXVte5DiKHTtyljb1jxT2Yt3SwvZSTJG72ycGfDmkmqSM6RNlK5qN9RD7OfPl-JDKPRbJi71WZ-P8aIBL05IiY0DMHreOo3j09v1vVwz7g3ORh5dA17wrLHLXjgi1wGC6J4ZT6EVxXlwIxeU7R1pqLhvfVhZk74GD5qqWJKoF5t3JypirP3tnBf6zSeaIOXvLurSEPP7QqrPPbSfa0m9aHnwvF0o0lt2SHWwRMBUWdyuLUVJv4GU7F45w\",\"e\":\"AQAB\",\"x5c\":[\"MIIClzCCAX8CBgGMH0942TANBgkqhkiG9w0BAQsFADAPMQ0wCwYDVQQDDAR0ZXN0MB4XDTIzMTEzMDA4MTczMVoXDTMzMTEzMDA4MTkxMVowDzENMAsGA1UEAwwEdGVzdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALlHcFirLvxMRwu/lcUvOGWzsSQVCvw5jGL3275rnz1+Vb9UYYNgzAdaMtYa89nCVQO3G5yqV7h4+QwugSdkbjvs2gvW1wZl1bXuQ4ih07cpY29Y8U9mLd0sL2UkyRu9snBnw5pJqkjOkTZSuajfUQ+znz5fiQyj0WyYu9Vmfj/GiAS9OSImNAzB63jqN49Pb9b1cM+4NzkYeXQNe8Kyxy144ItcBguieGU+hFcV5cCMXlO0daai4b31YWZO+Bg+aqliSqBebdycqYqz97ZwX+s0nmiDl7y7q0hDz+0Kqzz20n2tJvWh58LxdKNJbdkh1sETAVFncri1FSb+BlOxeOcCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAr7u7/VHUo6X2ig4JWAjeNVAcauYYLwnVpZpRKlqfx6GxyNZwGdxJSsfF4FTtfWODkLoOR6VJhQWzhpMiRIwrVIh6Oa9Qz1B8sbiM/AQA2WGI/ueLmcIzQdoOlOIZ6CiQA/P0virwrRu6ov0/ukOnuyLV9yRv08X4BnrfqIfIQTYzUhd0ylkF3l9lrrB1rxlGuWU0CA+jkj2D7JI7/0UPfHmbUzw1w70myHdnYzALnAWzQmHgWzmtMFmF0CW/EIanZ75/V6PMZ8+kOeBJfbJcD4C/fIYWIP174Uhu7xHNMQVtcnjUJ8si9VQgi9xR0kipXuu2UKFbPrMzHkqfNaOPxw==\"],\"x5t\":\"cDwkyVl_hhpJK69DIAxKqs_9vWE\",\"x5t#S256\":\"Y3hdo6JypgQC05Aa8qjkZi4me_JpEmsyTkc8n6Mkrcw\"},{\"kid\":\"ih_BbqHvev8DAqxCKAvIV6JI7A6L9fhwSBeDOJAtlIk\",\"kty\":\"RSA\",\"alg\":\"RSA-OAEP\",\"use\":\"enc\",\"n\":\"zCJQu4bWzdaHW5DJSJhKMKWjc6dNGYMGyPXYat90X4VixyQHrzYnH_0y0f0t88nmMClMrUawWr5oy7haok6IyVVFEhfviTMud0wLFCgNnS-rjSOwOsFk2hljo5NBjJCr-Y-MDv_r5vRdzmp1XKFLuqNplC6lB7JRiHGKZxHq3mXtFf7CtjspKRiPS5L1tKGfkJYdzEKL4jsOC53djWtJESTqvPINER4w0oDFR6cySc3Hrgu41wFov_v6g9lKt6Dj_yp5WrsDnO9yCfyS6Pg5wFWuMP6s0YG3v8e8E8Ss0ngO5ziEEFMolM2K-Orisq2koVDxbvHIipyIV3ZCofFVYQ\",\"e\":\"AQAB\",\"x5c\":[\"MIIClzCCAX8CBgGMH096ETANBgkqhkiG9w0BAQsFADAPMQ0wCwYDVQQDDAR0ZXN0MB4XDTIzMTEzMDA4MTczMVoXDTMzMTEzMDA4MTkxMVowDzENMAsGA1UEAwwEdGVzdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMwiULuG1s3Wh1uQyUiYSjClo3OnTRmDBsj12GrfdF+FYsckB682Jx/9MtH9LfPJ5jApTK1GsFq+aMu4WqJOiMlVRRIX74kzLndMCxQoDZ0vq40jsDrBZNoZY6OTQYyQq/mPjA7/6+b0Xc5qdVyhS7qjaZQupQeyUYhximcR6t5l7RX+wrY7KSkYj0uS9bShn5CWHcxCi+I7Dgud3Y1rSREk6rzyDREeMNKAxUenMknNx64LuNcBaL/7+oPZSreg4/8qeVq7A5zvcgn8kuj4OcBVrjD+rNGBt7/HvBPErNJ4Duc4hBBTKJTNivjq4rKtpKFQ8W7xyIqciFd2QqHxVWECAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAXkltYV9tfpK3STzLe0fyHwsVXhr8whCxBjleTVs3rmeNOjr8vjvB27NVI6jiOZZgj7oGTN9H8p8f4CeG+c8zZrwzxLdWRLrv1L7/B45RQbPGS5uYJ+Mcllqitx+y+YgVG+MHR0TJCjjpe8meYgAk7gCYtf+JAiRlqKqbmHuryqe1dGJ0v3ygDXM1HY3gKH1auE9kIDLnVbORzd0jRHjOjz72RpFFk+I7fJ05RRNyxmkVEKmpFT5K5DvrZizj1UNlsa8b8DWXwiWZDAiRpuNMR55hHrcj7S28TdTy+XfZVOTzJfAKuKQDfjgq1YuVq7EB9W0sx+gIvNvzxfnQiqpfdQ==\"],\"x5t\":\"KeyG6Ct9n6ufBknJClE5jS20LHU\",\"x5t#S256\":\"WYknXYM8JgRnURbarMyOeIITLgwSmfg69HCphgCb6mU\"}]}" - ))); + server.when( + request().withMethod("GET") + .withPath( + "/realms/" + OIDCMockServer.REALM_NAME + "/protocol/openid-connect/certs")) + .respond( + response().withBody( + JsonBody.json( + "{\"keys\":[{\"kid\":\"nW8YlwzTMqqOd6eqL1IXMZfBL7k2rSl8fX_NR80btvM\",\"kty\":\"RSA\",\"alg\":\"RS256\",\"use\":\"sig\",\"n\":\"uUdwWKsu_ExHC7-VxS84ZbOxJBUK_DmMYvfbvmufPX5Vv1Rhg2DMB1oy1hrz2cJVA7cbnKpXuHj5DC6BJ2RuO-zaC9bXBmXVte5DiKHTtyljb1jxT2Yt3SwvZSTJG72ycGfDmkmqSM6RNlK5qN9RD7OfPl-JDKPRbJi71WZ-P8aIBL05IiY0DMHreOo3j09v1vVwz7g3ORh5dA17wrLHLXjgi1wGC6J4ZT6EVxXlwIxeU7R1pqLhvfVhZk74GD5qqWJKoF5t3JypirP3tnBf6zSeaIOXvLurSEPP7QqrPPbSfa0m9aHnwvF0o0lt2SHWwRMBUWdyuLUVJv4GU7F45w\",\"e\":\"AQAB\",\"x5c\":[\"MIIClzCCAX8CBgGMH0942TANBgkqhkiG9w0BAQsFADAPMQ0wCwYDVQQDDAR0ZXN0MB4XDTIzMTEzMDA4MTczMVoXDTMzMTEzMDA4MTkxMVowDzENMAsGA1UEAwwEdGVzdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALlHcFirLvxMRwu/lcUvOGWzsSQVCvw5jGL3275rnz1+Vb9UYYNgzAdaMtYa89nCVQO3G5yqV7h4+QwugSdkbjvs2gvW1wZl1bXuQ4ih07cpY29Y8U9mLd0sL2UkyRu9snBnw5pJqkjOkTZSuajfUQ+znz5fiQyj0WyYu9Vmfj/GiAS9OSImNAzB63jqN49Pb9b1cM+4NzkYeXQNe8Kyxy144ItcBguieGU+hFcV5cCMXlO0daai4b31YWZO+Bg+aqliSqBebdycqYqz97ZwX+s0nmiDl7y7q0hDz+0Kqzz20n2tJvWh58LxdKNJbdkh1sETAVFncri1FSb+BlOxeOcCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAr7u7/VHUo6X2ig4JWAjeNVAcauYYLwnVpZpRKlqfx6GxyNZwGdxJSsfF4FTtfWODkLoOR6VJhQWzhpMiRIwrVIh6Oa9Qz1B8sbiM/AQA2WGI/ueLmcIzQdoOlOIZ6CiQA/P0virwrRu6ov0/ukOnuyLV9yRv08X4BnrfqIfIQTYzUhd0ylkF3l9lrrB1rxlGuWU0CA+jkj2D7JI7/0UPfHmbUzw1w70myHdnYzALnAWzQmHgWzmtMFmF0CW/EIanZ75/V6PMZ8+kOeBJfbJcD4C/fIYWIP174Uhu7xHNMQVtcnjUJ8si9VQgi9xR0kipXuu2UKFbPrMzHkqfNaOPxw==\"],\"x5t\":\"cDwkyVl_hhpJK69DIAxKqs_9vWE\",\"x5t#S256\":\"Y3hdo6JypgQC05Aa8qjkZi4me_JpEmsyTkc8n6Mkrcw\"},{\"kid\":\"ih_BbqHvev8DAqxCKAvIV6JI7A6L9fhwSBeDOJAtlIk\",\"kty\":\"RSA\",\"alg\":\"RSA-OAEP\",\"use\":\"enc\",\"n\":\"zCJQu4bWzdaHW5DJSJhKMKWjc6dNGYMGyPXYat90X4VixyQHrzYnH_0y0f0t88nmMClMrUawWr5oy7haok6IyVVFEhfviTMud0wLFCgNnS-rjSOwOsFk2hljo5NBjJCr-Y-MDv_r5vRdzmp1XKFLuqNplC6lB7JRiHGKZxHq3mXtFf7CtjspKRiPS5L1tKGfkJYdzEKL4jsOC53djWtJESTqvPINER4w0oDFR6cySc3Hrgu41wFov_v6g9lKt6Dj_yp5WrsDnO9yCfyS6Pg5wFWuMP6s0YG3v8e8E8Ss0ngO5ziEEFMolM2K-Orisq2koVDxbvHIipyIV3ZCofFVYQ\",\"e\":\"AQAB\",\"x5c\":[\"MIIClzCCAX8CBgGMH096ETANBgkqhkiG9w0BAQsFADAPMQ0wCwYDVQQDDAR0ZXN0MB4XDTIzMTEzMDA4MTczMVoXDTMzMTEzMDA4MTkxMVowDzENMAsGA1UEAwwEdGVzdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMwiULuG1s3Wh1uQyUiYSjClo3OnTRmDBsj12GrfdF+FYsckB682Jx/9MtH9LfPJ5jApTK1GsFq+aMu4WqJOiMlVRRIX74kzLndMCxQoDZ0vq40jsDrBZNoZY6OTQYyQq/mPjA7/6+b0Xc5qdVyhS7qjaZQupQeyUYhximcR6t5l7RX+wrY7KSkYj0uS9bShn5CWHcxCi+I7Dgud3Y1rSREk6rzyDREeMNKAxUenMknNx64LuNcBaL/7+oPZSreg4/8qeVq7A5zvcgn8kuj4OcBVrjD+rNGBt7/HvBPErNJ4Duc4hBBTKJTNivjq4rKtpKFQ8W7xyIqciFd2QqHxVWECAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAXkltYV9tfpK3STzLe0fyHwsVXhr8whCxBjleTVs3rmeNOjr8vjvB27NVI6jiOZZgj7oGTN9H8p8f4CeG+c8zZrwzxLdWRLrv1L7/B45RQbPGS5uYJ+Mcllqitx+y+YgVG+MHR0TJCjjpe8meYgAk7gCYtf+JAiRlqKqbmHuryqe1dGJ0v3ygDXM1HY3gKH1auE9kIDLnVbORzd0jRHjOjz72RpFFk+I7fJ05RRNyxmkVEKmpFT5K5DvrZizj1UNlsa8b8DWXwiWZDAiRpuNMR55hHrcj7S28TdTy+XfZVOTzJfAKuKQDfjgq1YuVq7EB9W0sx+gIvNvzxfnQiqpfdQ==\"],\"x5t\":\"KeyG6Ct9n6ufBknJClE5jS20LHU\",\"x5t#S256\":\"WYknXYM8JgRnURbarMyOeIITLgwSmfg69HCphgCb6mU\"}]}" + ))); }); - REALM_FACTORY.setWellKnownEndpoint(URI.create(OIDC_SERVER.baseUrl() - + "/realms/" + OIDCMockServer.REALM_NAME + "/.well-known/uma2-configuration")); + REALM_FACTORY.setWellKnownEndpoint( + URI.create( + OIDC_SERVER.baseUrl() + "/realms/" + OIDCMockServer.REALM_NAME + "/.well-known/uma2-configuration")); } @Test diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealmTest.java b/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealmTest.java index a86999d04a2..71edbd81478 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealmTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealmTest.java @@ -60,13 +60,21 @@ static void setup() throws NoSuchAlgorithmException { // Create the realm REALM = new JwtPkceVerifyingRealm( - () -> Optional.of(new JwtPkceVerifyingRealmFactory.IdpConfiguration(Map.of(KEY_ID, PUBLIC_KEY), URI.create("auth"), URI.create("token"), URI.create("logout"), HTTP_REALM_URL)), - AUDIENCE, - List.of(JwtPkceVerifyingRealmFactory.ScriptedTokenChecker.create("t.getOtherClaims().get(\"groups\").equals(\"conquery\")")), - List.of(ALTERNATIVE_ID_CLAIM), - STORAGE, - TOKEN_LEEWAY, - BaseValidator.newValidator() + () -> Optional.of( + new JwtPkceVerifyingRealmFactory.IdpConfiguration( + Map.of(KEY_ID, PUBLIC_KEY), + URI.create("auth"), + URI.create("token"), + URI.create("logout"), + HTTP_REALM_URL)), + AUDIENCE, + List.of( + JwtPkceVerifyingRealmFactory.ScriptedTokenChecker.create( + "t.getOtherClaims().get(\"groups\").equals(\"conquery\")")), + List.of(ALTERNATIVE_ID_CLAIM), + STORAGE, + TOKEN_LEEWAY, + BaseValidator.newValidator() ); } @@ -86,20 +94,25 @@ void verifyToken() { Date issueDate = new Date(); Date expDate = DateUtils.addMinutes(issueDate, 1); String token = JWT.create() - .withIssuer(HTTP_REALM_URL) - .withAudience(AUDIENCE) - .withSubject(expected.getName()) - .withIssuedAt(issueDate) - .withExpiresAt(expDate) - .withClaim("groups", "conquery") - .withIssuedAt(issueDate) - .withExpiresAt(expDate) - .withKeyId(KEY_ID) - .withJWTId(UUID.randomUUID().toString()) - .sign(Algorithm.RSA256(PUBLIC_KEY, PRIVATE_KEY)); + .withIssuer(HTTP_REALM_URL) + .withAudience(AUDIENCE) + .withSubject( + expected.getName()) + .withIssuedAt(issueDate) + .withExpiresAt(expDate) + .withClaim( + "groups", + "conquery") + .withIssuedAt(issueDate) + .withExpiresAt(expDate) + .withKeyId(KEY_ID) + .withJWTId( + UUID.randomUUID().toString()) + .sign(Algorithm.RSA256(PUBLIC_KEY, PRIVATE_KEY)); BearerToken accessToken = new BearerToken(token); - assertThat(REALM.doGetAuthenticationInfo(accessToken).getPrincipals().getPrimaryPrincipal()).isEqualTo(expected); + assertThat(REALM.doGetAuthenticationInfo(accessToken).getPrincipals().getPrimaryPrincipal()).isEqualTo( + expected); } @Test @@ -115,22 +128,27 @@ void verifyTokenAndAddRole() { Date issueDate = new Date(); Date expDate = DateUtils.addMinutes(issueDate, 1); String token = JWT.create() - .withIssuer(HTTP_REALM_URL) - .withAudience(AUDIENCE) - .withSubject(expected.getName()) - .withIssuedAt(issueDate) - .withExpiresAt(expDate) - .withClaim("groups", "conquery") - .withClaim("resource_access", Map.of(AUDIENCE, Map.of("roles", List.of("admin", "unknown")))) // See structure of AccessToken.Access - .withIssuedAt(issueDate) - .withExpiresAt(expDate) - .withKeyId(KEY_ID) - .withJWTId(UUID.randomUUID().toString()) - .sign(Algorithm.RSA256(PUBLIC_KEY, PRIVATE_KEY)); + .withIssuer(HTTP_REALM_URL) + .withAudience(AUDIENCE) + .withSubject( + expected.getName()) + .withIssuedAt(issueDate) + .withExpiresAt(expDate) + .withClaim( + "groups", + "conquery") + .withClaim("resource_access", Map.of(AUDIENCE, Map.of("roles", List.of("admin", "unknown")))) // See structure of AccessToken.Access + .withIssuedAt(issueDate) + .withExpiresAt(expDate) + .withKeyId(KEY_ID) + .withJWTId( + UUID.randomUUID().toString()) + .sign(Algorithm.RSA256(PUBLIC_KEY, PRIVATE_KEY)); BearerToken accessToken = new BearerToken(token); - assertThat(REALM.doGetAuthenticationInfo(accessToken).getPrincipals().getPrimaryPrincipal()).isEqualTo(expected); + assertThat(REALM.doGetAuthenticationInfo(accessToken).getPrincipals().getPrimaryPrincipal()).isEqualTo( + expected); assertThat(expected.getRoles()).contains(role.getId()); } @@ -146,23 +164,28 @@ void verifyTokenAndAddRoleNewUser() { Date issueDate = new Date(); Date expDate = DateUtils.addMinutes(issueDate, 1); String token = JWT.create() - .withClaim(IDToken.NAME, "New User") - .withIssuer(HTTP_REALM_URL) - .withAudience(AUDIENCE) - .withSubject(expected.getName()) - .withIssuedAt(issueDate) - .withExpiresAt(expDate) - .withClaim("groups", "conquery") - .withClaim("resource_access", Map.of(AUDIENCE, Map.of("roles", List.of("admin", "unknown")))) // See structure of AccessToken.Access - .withIssuedAt(issueDate) - .withExpiresAt(expDate) - .withKeyId(KEY_ID) - .withJWTId(UUID.randomUUID().toString()) - .sign(Algorithm.RSA256(PUBLIC_KEY, PRIVATE_KEY)); + .withClaim(IDToken.NAME, "New User") + .withIssuer(HTTP_REALM_URL) + .withAudience( + AUDIENCE) + .withSubject(expected.getName()) + .withIssuedAt(issueDate) + .withExpiresAt(expDate) + .withClaim( + "groups", + "conquery") + .withClaim("resource_access", Map.of(AUDIENCE, Map.of("roles", List.of("admin", "unknown")))) // See structure of AccessToken.Access + .withIssuedAt(issueDate) + .withExpiresAt(expDate) + .withKeyId(KEY_ID) + .withJWTId( + UUID.randomUUID().toString()) + .sign(Algorithm.RSA256(PUBLIC_KEY, PRIVATE_KEY)); BearerToken accessToken = new BearerToken(token); - assertThat(REALM.doGetAuthenticationInfo(accessToken).getPrincipals().getPrimaryPrincipal().getId()).isEqualTo(expected.getId()); + assertThat(REALM.doGetAuthenticationInfo(accessToken).getPrincipals().getPrimaryPrincipal().getId()).isEqualTo( + expected.getId()); assertThat(STORAGE.getUser(expected.getId()).getRoles()).contains(role.getId()); assertThat(STORAGE.getUser(expected.getId()).getLabel()).isEqualTo(expected.getLabel()); } @@ -180,20 +203,25 @@ void verifyTokenInLeeway() { Date expDate = DateUtils.addSeconds(issueDate, -TOKEN_LEEWAY / 2); String token = JWT.create() - .withIssuer(HTTP_REALM_URL) - .withAudience(AUDIENCE) - .withSubject(expected.getName()) - .withIssuedAt(issueDate) - .withExpiresAt(expDate) - .withClaim("groups", "conquery") - .withIssuedAt(issueDate) - .withExpiresAt(expDate) - .withKeyId(KEY_ID) - .withJWTId(UUID.randomUUID().toString()) - .sign(Algorithm.RSA256(PUBLIC_KEY, PRIVATE_KEY)); + .withIssuer(HTTP_REALM_URL) + .withAudience(AUDIENCE) + .withSubject( + expected.getName()) + .withIssuedAt(issueDate) + .withExpiresAt(expDate) + .withClaim( + "groups", + "conquery") + .withIssuedAt(issueDate) + .withExpiresAt(expDate) + .withKeyId(KEY_ID) + .withJWTId( + UUID.randomUUID().toString()) + .sign(Algorithm.RSA256(PUBLIC_KEY, PRIVATE_KEY)); BearerToken accessToken = new BearerToken(token); - assertThat(REALM.doGetAuthenticationInfo(accessToken).getPrincipals().getPrimaryPrincipal()).isEqualTo(expected); + assertThat(REALM.doGetAuthenticationInfo(accessToken).getPrincipals().getPrimaryPrincipal()).isEqualTo( + expected); } @@ -208,20 +236,26 @@ void verifyTokenAlternativeId() { Date expDate = DateUtils.addMinutes(issueDate, 1); final String primId = UUID.randomUUID().toString(); String token = JWT.create() - .withIssuer(HTTP_REALM_URL) - .withAudience(AUDIENCE) - .withSubject(primId) - .withClaim("groups", "conquery") - .withIssuedAt(issueDate) - .withExpiresAt(expDate) - .withClaim(ALTERNATIVE_ID_CLAIM, expected.getName()) - .withKeyId(KEY_ID) - .withJWTId(UUID.randomUUID().toString()) - .sign(Algorithm.RSA256(PUBLIC_KEY, PRIVATE_KEY)); + .withIssuer(HTTP_REALM_URL) + .withAudience(AUDIENCE) + .withSubject(primId) + .withClaim( + "groups", + "conquery") + .withIssuedAt(issueDate) + .withExpiresAt(expDate) + .withClaim( + ALTERNATIVE_ID_CLAIM, + expected.getName()) + .withKeyId(KEY_ID) + .withJWTId(UUID.randomUUID().toString()) + .sign( + Algorithm.RSA256(PUBLIC_KEY, PRIVATE_KEY)); BearerToken accessToken = new BearerToken(token); - assertThat(REALM.doGetAuthenticationInfo(accessToken).getPrincipals().getPrimaryPrincipal()).isEqualTo(expected); + assertThat(REALM.doGetAuthenticationInfo(accessToken).getPrincipals().getPrimaryPrincipal()).isEqualTo( + expected); } @@ -234,16 +268,19 @@ void falsifyTokenMissingCustomClaim() { Date issueDate = new Date(); Date expDate = DateUtils.addMinutes(issueDate, 1); String token = JWT.create() - .withIssuer(HTTP_REALM_URL) - .withAudience(AUDIENCE) - .withSubject(expected.getName()) - .withIssuedAt(issueDate) - .withExpiresAt(expDate) - .withKeyId(KEY_ID) - .sign(Algorithm.RSA256(PUBLIC_KEY, PRIVATE_KEY)); + .withIssuer(HTTP_REALM_URL) + .withAudience(AUDIENCE) + .withSubject( + expected.getName()) + .withIssuedAt(issueDate) + .withExpiresAt(expDate) + .withKeyId(KEY_ID) + .sign( + Algorithm.RSA256(PUBLIC_KEY, PRIVATE_KEY)); BearerToken accessToken = new BearerToken(token); - assertThatCode(() -> REALM.doGetAuthenticationInfo(accessToken)).hasCauseInstanceOf(VerificationException.class); + assertThatCode(() -> REALM.doGetAuthenticationInfo(accessToken)).hasCauseInstanceOf( + VerificationException.class); } @Test @@ -255,18 +292,22 @@ void falsifyTokenWrongAudience() { Date issueDate = new Date(); Date expDate = DateUtils.addMinutes(issueDate, 1); String token = JWT.create() - .withIssuer(HTTP_REALM_URL) - .withAudience("wrong_aud") - .withSubject(expected.getName()) - .withClaim("groups", "conquery") - .withIssuedAt(issueDate) - .withExpiresAt(expDate) - .withKeyId(KEY_ID) - .withJWTId(UUID.randomUUID().toString()) - .sign(Algorithm.RSA256(PUBLIC_KEY, PRIVATE_KEY)); + .withIssuer(HTTP_REALM_URL) + .withAudience("wrong_aud") + .withSubject( + expected.getName()) + .withClaim("groups", "conquery") + .withIssuedAt(issueDate) + .withExpiresAt( + expDate) + .withKeyId(KEY_ID) + .withJWTId(UUID.randomUUID().toString()) + .sign( + Algorithm.RSA256(PUBLIC_KEY, PRIVATE_KEY)); BearerToken accessToken = new BearerToken(token); - assertThatCode(() -> REALM.doGetAuthenticationInfo(accessToken)).hasCauseInstanceOf(VerificationException.class); + assertThatCode(() -> REALM.doGetAuthenticationInfo(accessToken)).hasCauseInstanceOf( + VerificationException.class); } @Test @@ -277,17 +318,21 @@ void falsifyTokenOutdated() { Date issueDate = new Date(); Date expDate = DateUtils.addMinutes(issueDate, -2); String token = JWT.create() - .withIssuer(HTTP_REALM_URL) - .withSubject(expected.getName()) - .withClaim("groups", "conquery") - .withIssuedAt(issueDate) - .withExpiresAt(expDate) - .withKeyId(KEY_ID) - .withJWTId(UUID.randomUUID().toString()) - .sign(Algorithm.RSA256(PUBLIC_KEY, PRIVATE_KEY)); + .withIssuer(HTTP_REALM_URL) + .withSubject(expected.getName()) + .withClaim( + "groups", + "conquery") + .withIssuedAt(issueDate) + .withExpiresAt(expDate) + .withKeyId(KEY_ID) + .withJWTId( + UUID.randomUUID().toString()) + .sign(Algorithm.RSA256(PUBLIC_KEY, PRIVATE_KEY)); BearerToken accessToken = new BearerToken(token); - assertThatCode(() -> REALM.doGetAuthenticationInfo(accessToken)).hasCauseInstanceOf(VerificationException.class); + assertThatCode(() -> REALM.doGetAuthenticationInfo(accessToken)).hasCauseInstanceOf( + VerificationException.class); } @Test @@ -299,20 +344,25 @@ void falsifyTokenWrongIssuer() { Date issueDate = new Date(); Date expDate = DateUtils.addMinutes(issueDate, 1); String token = JWT.create() - .withIssuer("wrong_iss") - .withAudience(AUDIENCE) - .withSubject(expected.getName()) - .withIssuedAt(issueDate) - .withExpiresAt(expDate) - .withClaim("groups", "conquery") - .withIssuedAt(issueDate) - .withExpiresAt(expDate) - .withKeyId(KEY_ID) - .withJWTId(UUID.randomUUID().toString()) - .sign(Algorithm.RSA256(PUBLIC_KEY, PRIVATE_KEY)); + .withIssuer("wrong_iss") + .withAudience(AUDIENCE) + .withSubject( + expected.getName()) + .withIssuedAt(issueDate) + .withExpiresAt(expDate) + .withClaim( + "groups", + "conquery") + .withIssuedAt(issueDate) + .withExpiresAt(expDate) + .withKeyId(KEY_ID) + .withJWTId( + UUID.randomUUID().toString()) + .sign(Algorithm.RSA256(PUBLIC_KEY, PRIVATE_KEY)); BearerToken accessToken = new BearerToken(token); - assertThatCode(() -> REALM.doGetAuthenticationInfo(accessToken)).hasCauseInstanceOf(VerificationException.class); + assertThatCode(() -> REALM.doGetAuthenticationInfo(accessToken)).hasCauseInstanceOf( + VerificationException.class); } @Test @@ -325,19 +375,23 @@ void falsifyTokenUnknownKid() { Date issueDate = new Date(); Date expDate = DateUtils.addMinutes(issueDate, 1); String token = JWT.create() - .withIssuer(HTTP_REALM_URL) - .withAudience(AUDIENCE) - .withSubject(expected.getName()) - .withIssuedAt(issueDate) - .withExpiresAt(expDate) - .withClaim("groups", "conquery") - .withIssuedAt(issueDate) - .withExpiresAt(expDate) - .withKeyId("unknown_key_id") - .withJWTId(UUID.randomUUID().toString()) - .sign(Algorithm.RSA256(PUBLIC_KEY, PRIVATE_KEY)); + .withIssuer(HTTP_REALM_URL) + .withAudience(AUDIENCE) + .withSubject( + expected.getName()) + .withIssuedAt(issueDate) + .withExpiresAt(expDate) + .withClaim( + "groups", + "conquery") + .withIssuedAt(issueDate) + .withExpiresAt(expDate) + .withKeyId("unknown_key_id") + .withJWTId( + UUID.randomUUID().toString()) + .sign(Algorithm.RSA256(PUBLIC_KEY, PRIVATE_KEY)); BearerToken accessToken = new BearerToken(token); assertThatCode(() -> REALM.doGetAuthenticationInfo(accessToken)).isInstanceOf(UnsupportedTokenException.class); } -} \ No newline at end of file +} diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/KeycloakApiTest.java b/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/KeycloakApiTest.java index 529659342ae..58e1f197f46 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/KeycloakApiTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/KeycloakApiTest.java @@ -2,11 +2,10 @@ import static org.assertj.core.api.Assertions.assertThat; +import jakarta.ws.rs.client.ClientBuilder; import java.util.Map; import java.util.Set; -import jakarta.ws.rs.client.ClientBuilder; - import com.bakdata.conquery.models.auth.oidc.keycloak.KeycloakApi; import com.bakdata.conquery.models.auth.oidc.keycloak.KeycloakGroup; import com.bakdata.conquery.models.config.auth.IntrospectionDelegatingRealmFactory; @@ -16,7 +15,10 @@ import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; @Slf4j -@EnabledIfEnvironmentVariable(named = "KEYCLOAK_SERVER_URL", matches = "^https?://.+", disabledReason = "As long as there is no mock server, test manually against a real server") +@EnabledIfEnvironmentVariable( + named = "KEYCLOAK_SERVER_URL", + matches = "^https?://.+", + disabledReason = "As long as there is no mock server, test manually against a real server") @EnabledIfEnvironmentVariable(named = "KEYCLOAK_REALM", matches = ".+") @EnabledIfEnvironmentVariable(named = "KEYCLOAK_CLIENT_ID", matches = ".+") @EnabledIfEnvironmentVariable(named = "KEYCLOAK_CLIENT_SECRET", matches = ".+") diff --git a/backend/src/test/java/com/bakdata/conquery/models/common/CQuarterTest.java b/backend/src/test/java/com/bakdata/conquery/models/common/CQuarterTest.java index cea8970b460..49c096ca058 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/common/CQuarterTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/common/CQuarterTest.java @@ -8,28 +8,27 @@ import java.util.List; import java.util.stream.IntStream; +import com.google.common.primitives.Ints; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import com.google.common.primitives.Ints; - class CQuarterTest { public static List testLastDay() { List values = new ArrayList<>(); - IntStream.range(1946, 2100) - .forEach(year -> { - for(int quarter=1;quarter<=4;quarter++) { - LocalDate realFirst = LocalDate.of(year, (quarter-1)*3+1, 1); - LocalDate realLast = LocalDate.of(year, quarter*3, Month.of(quarter*3).length(false)); + IntStream.range(1946, 2100).forEach(year -> { + for (int quarter = 1; quarter <= 4; quarter++) { + LocalDate realFirst = LocalDate.of(year, (quarter - 1) * 3 + 1, 1); + LocalDate realLast = LocalDate.of(year, quarter * 3, Month.of(quarter * 3).length(false)); values.add(Arguments.of(realFirst, realLast)); } }); return values; } - - @ParameterizedTest(name = "{0}") @MethodSource + + @ParameterizedTest(name = "{0}") + @MethodSource public void testLastDay(LocalDate first, LocalDate last) { int firstEpoch = Ints.checkedCast(first.toEpochDay()); int lastEpoch = Ints.checkedCast(last.toEpochDay()); diff --git a/backend/src/test/java/com/bakdata/conquery/models/common/QuarterUtilsTest.java b/backend/src/test/java/com/bakdata/conquery/models/common/QuarterUtilsTest.java index 12934047f84..e7e0333990e 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/common/QuarterUtilsTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/common/QuarterUtilsTest.java @@ -14,51 +14,51 @@ class QuarterUtilsTest { void nextQuarterAdjuster() { // YearMonths - assertThat(QuarterUtils.nextQuarterAdjuster().adjustInto(YearMonth.of(2018, 1))) - .isEqualTo(YearMonth.of(2018, 4)); - assertThat(QuarterUtils.nextQuarterAdjuster().adjustInto(YearMonth.of(2018, 2))) - .isEqualTo(YearMonth.of(2018, 4)); + assertThat(QuarterUtils.nextQuarterAdjuster().adjustInto(YearMonth.of(2018, 1))).isEqualTo( + YearMonth.of(2018, 4)); + assertThat(QuarterUtils.nextQuarterAdjuster().adjustInto(YearMonth.of(2018, 2))).isEqualTo( + YearMonth.of(2018, 4)); - assertThat(QuarterUtils.nextQuarterAdjuster().adjustInto(YearMonth.of(2018, 4))) - .isEqualTo(YearMonth.of(2018, 7)); + assertThat(QuarterUtils.nextQuarterAdjuster().adjustInto(YearMonth.of(2018, 4))).isEqualTo( + YearMonth.of(2018, 7)); - assertThat(QuarterUtils.nextQuarterAdjuster().adjustInto(YearMonth.of(2018, 6))) - .isEqualTo(YearMonth.of(2018, 7)); + assertThat(QuarterUtils.nextQuarterAdjuster().adjustInto(YearMonth.of(2018, 6))).isEqualTo( + YearMonth.of(2018, 7)); // LocalDates - assertThat(QuarterUtils.nextQuarterAdjuster().adjustInto(LocalDate.of(2018, 6, 25))) - .isEqualTo(LocalDate.of(2018, 7, 25)); + assertThat(QuarterUtils.nextQuarterAdjuster().adjustInto(LocalDate.of(2018, 6, 25))).isEqualTo( + LocalDate.of(2018, 7, 25)); - assertThat(QuarterUtils.nextQuarterAdjuster().adjustInto(LocalDate.of(2018, 12, 25))) - .isEqualTo(LocalDate.of(2019, 1, 25)); + assertThat(QuarterUtils.nextQuarterAdjuster().adjustInto(LocalDate.of(2018, 12, 25))).isEqualTo( + LocalDate.of(2019, 1, 25)); } @Test void firstMonthInQuarterAdjuster() { - assertThat(QuarterUtils.firstMonthInQuarterAdjuster().adjustInto(LocalDate.of(2018, 12, 25))) - .isEqualTo(LocalDate.of(2018, 10, 25)); + assertThat(QuarterUtils.firstMonthInQuarterAdjuster().adjustInto(LocalDate.of(2018, 12, 25))).isEqualTo( + LocalDate.of(2018, 10, 25)); - assertThat(QuarterUtils.firstMonthInQuarterAdjuster().adjustInto(LocalDateTime.of(2018, 2, 5, 5, 5))) - .isEqualTo(LocalDateTime.of(2018, 1, 5, 5, 5)); + assertThat(QuarterUtils.firstMonthInQuarterAdjuster().adjustInto(LocalDateTime.of(2018, 2, 5, 5, 5))).isEqualTo( + LocalDateTime.of(2018, 1, 5, 5, 5)); } @Test void firstDayOfQuarterAdjuster() { - assertThat(QuarterUtils.firstDayOfQuarterAdjuster().adjustInto(LocalDate.of(2018, 12, 25))) - .isEqualTo(LocalDate.of(2018, 10, 1)); + assertThat(QuarterUtils.firstDayOfQuarterAdjuster().adjustInto(LocalDate.of(2018, 12, 25))).isEqualTo( + LocalDate.of(2018, 10, 1)); - assertThat(QuarterUtils.firstDayOfQuarterAdjuster().adjustInto(LocalDateTime.of(2018, 2, 5, 5, 5))) - .isEqualTo(LocalDateTime.of(2018, 1, 1, 5, 5)); + assertThat(QuarterUtils.firstDayOfQuarterAdjuster().adjustInto(LocalDateTime.of(2018, 2, 5, 5, 5))).isEqualTo( + LocalDateTime.of(2018, 1, 1, 5, 5)); } @Test void lastDayOfQuarterAdjuster() { - assertThat(QuarterUtils.lastDayOfQuarterAdjuster().adjustInto(LocalDate.of(2018, 12, 25))) - .isEqualTo(LocalDate.of(2018, 12, 31)); + assertThat(QuarterUtils.lastDayOfQuarterAdjuster().adjustInto(LocalDate.of(2018, 12, 25))).isEqualTo( + LocalDate.of(2018, 12, 31)); - assertThat(QuarterUtils.lastDayOfQuarterAdjuster().adjustInto(LocalDateTime.of(2018, 2, 5, 5, 5))) - .isEqualTo(LocalDateTime.of(2018, 3, 31, 5, 5)); + assertThat(QuarterUtils.lastDayOfQuarterAdjuster().adjustInto(LocalDateTime.of(2018, 2, 5, 5, 5))).isEqualTo( + LocalDateTime.of(2018, 3, 31, 5, 5)); } -} \ No newline at end of file +} diff --git a/backend/src/test/java/com/bakdata/conquery/models/common/RangeTest.java b/backend/src/test/java/com/bakdata/conquery/models/common/RangeTest.java index 0bfb7d2e647..6f587727512 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/common/RangeTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/common/RangeTest.java @@ -26,65 +26,91 @@ class RangeTest { public void exactly() { Range exactly = Range.exactly(5); - assertThat((IntPredicate) exactly::contains) - .accepts(5) - .rejects(6, 7, 4); + assertThat((IntPredicate) exactly::contains).accepts(5).rejects(6, 7, 4); - assertThat((Predicate>) exactly::contains) - .accepts(Range.exactly(5), exactly) - .rejects(Range.exactly(6), Range.exactly(7), Range.exactly(4)); + assertThat((Predicate>) exactly::contains).accepts(Range.exactly(5), exactly) + .rejects( + Range.exactly(6), + Range.exactly(7), + Range.exactly(4)); } @Test public void atMost() { Range range = Range.atMost(5); - assertThat((IntPredicate) range::contains) - .accepts(5, 4, CDateRange.NEGATIVE_INFINITY) - .rejects(6, CDateRange.POSITIVE_INFINITY); + assertThat((IntPredicate) range::contains).accepts(5, 4, CDateRange.NEGATIVE_INFINITY) + .rejects( + 6, + CDateRange.POSITIVE_INFINITY); - assertThat((Predicate>) range::contains) - .accepts(Range.exactly(5), Range.atMost(4)) - .rejects(Range.exactly(6), Range.atLeast(5), Range.atLeast(4)); + assertThat((Predicate>) range::contains).accepts(Range.exactly(5), Range.atMost(4)) + .rejects( + Range.exactly(6), + Range.atLeast(5), + Range.atLeast(4)); } @Test public void atLeast() { Range range = Range.atLeast(5); - assertThat((IntPredicate) range::contains) - .accepts(5, 6, CDateRange.POSITIVE_INFINITY) - .rejects(4, CDateRange.NEGATIVE_INFINITY) - ; + assertThat((IntPredicate) range::contains).accepts(5, 6, CDateRange.POSITIVE_INFINITY) + .rejects( + 4, + CDateRange.NEGATIVE_INFINITY); - assertThat((Predicate>) range::contains) - .rejects(Range.atLeast(4), Range.atMost(4), Range.exactly(4)) - .accepts(Range.exactly(5), Range.exactly(6), Range.atLeast(5), Range.atLeast(6)); + assertThat((Predicate>) range::contains).rejects( + Range.atLeast(4), + Range.atMost(4), + Range.exactly(4)).accepts(Range.exactly(5), Range.exactly(6), Range.atLeast(5), Range.atLeast(6)); } @Test public void all() { Range range = Range.all(); - assertThat((IntPredicate) range::contains) - .accepts(5, CDateRange.POSITIVE_INFINITY, CDateRange.NEGATIVE_INFINITY); + assertThat((IntPredicate) range::contains).accepts( + 5, + CDateRange.POSITIVE_INFINITY, + CDateRange.NEGATIVE_INFINITY); - assertThat((Predicate>) range::contains) - .accepts(Range.exactly(5), Range.of(5, 10)) - .rejects(Range.atMost(5), Range.atLeast(6)); + assertThat((Predicate>) range::contains).accepts(Range.exactly(5), Range.of(5, 10)) + .rejects( + Range.atMost(5), + Range.atLeast(6)); } @Test public void contains() { Range range = Range.of(5, 10); - assertThat((IntPredicate) range::contains) - .accepts(5, 6, 7, 8, 9, 10) - .rejects(CDateRange.NEGATIVE_INFINITY, 4, 11, CDateRange.POSITIVE_INFINITY); - - assertThat((Predicate>) range::contains) - .accepts(range, Range.of(5, 10), Range.exactly(7), Range.exactly(5), Range.exactly(10), Range.of(6, 9), Range.of(5, 9)) - .rejects(Range.atMost(5), Range.atMost(10), Range.all(), Range.exactly(4), Range.exactly(11), Range.atLeast(5), Range.atLeast(10), Range.of(7, 11), Range.of(11, 12), Range.of(3, 4)); + assertThat((IntPredicate) range::contains).accepts(5, 6, 7, 8, 9, 10) + .rejects( + CDateRange.NEGATIVE_INFINITY, + 4, + 11, + CDateRange.POSITIVE_INFINITY); + + assertThat((Predicate>) range::contains).accepts( + range, + Range.of(5, 10), + Range.exactly(7), + Range.exactly(5), + Range.exactly(10), + Range.of(6, 9), + Range.of(5, 9)) + .rejects( + Range.atMost(5), + Range.atMost(10), + Range.all(), + Range.exactly(4), + Range.exactly(11), + Range.atLeast(5), + Range.atLeast(10), + Range.of(7, 11), + Range.of(11, 12), + Range.of(3, 4)); } @Test @@ -94,18 +120,17 @@ public void invalid() { @Test public void span() { - assertThat(Range.exactly(5).span(Range.exactly(6))) - .isEqualTo(Range.of(5, 6)) - .isEqualTo(Range.exactly(6).span(Range.exactly(5))); + assertThat(Range.exactly(5).span(Range.exactly(6))).isEqualTo(Range.of(5, 6)) + .isEqualTo( + Range.exactly(6).span(Range.exactly(5))); - assertThat(Range.of(5, 7).span(Range.exactly(6))) - .isEqualTo(Range.of(5, 7)); + assertThat(Range.of(5, 7).span(Range.exactly(6))).isEqualTo(Range.of(5, 7)); } - + @Test public void coveredYears() { CDateRange dateRange = CDateRange.of(LocalDate.of(2000, 9, 2), LocalDate.of(2005, 3, 15)); - + List expected = new ArrayList<>(); expected.add(CDateRange.of(LocalDate.of(2000, 9, 2), LocalDate.of(2000, 12, 31))); expected.add(CDateRange.of(LocalDate.of(2001, 1, 1), LocalDate.of(2001, 12, 31))); @@ -113,7 +138,7 @@ public void coveredYears() { expected.add(CDateRange.of(LocalDate.of(2003, 1, 1), LocalDate.of(2003, 12, 31))); expected.add(CDateRange.of(LocalDate.of(2004, 1, 1), LocalDate.of(2004, 12, 31))); expected.add(CDateRange.of(LocalDate.of(2005, 1, 1), LocalDate.of(2005, 3, 15))); - + assertThat(dateRange.getCoveredYears()).containsExactlyInAnyOrderElementsOf(expected); } @@ -137,31 +162,32 @@ public void coveredQuarters() { public void coveredQuartersNotAFullQuarter() { CDateRange dateRange = CDateRange.of(LocalDate.of(2000, 1, 10), LocalDate.of(2000, 3, 15)); - assertThat(dateRange.getCoveredQuarters()).containsExactlyInAnyOrder(CDateRange.of(LocalDate.of(2000, 1, 10), LocalDate.of(2000, 3, 15))); + assertThat(dateRange.getCoveredQuarters()).containsExactlyInAnyOrder( + CDateRange.of(LocalDate.of(2000, 1, 10), LocalDate.of(2000, 3, 15))); } - + public static List deserialize() { return Arrays.asList( Arguments.of( - "{\"min\":\"2017-01-01\", \"max\":\"2017-01-01\"}", - new Range<>(LocalDate.of(2017, 1, 1), LocalDate.of(2017, 1, 1)), - CDateRange.of(LocalDate.of(2017, 1, 1), LocalDate.of(2017, 1, 1)) + "{\"min\":\"2017-01-01\", \"max\":\"2017-01-01\"}", + new Range<>(LocalDate.of(2017, 1, 1), LocalDate.of(2017, 1, 1)), + CDateRange.of(LocalDate.of(2017, 1, 1), LocalDate.of(2017, 1, 1)) ), Arguments.of( - "{\"min\":\"2017-01-01\"}", - new Range<>(LocalDate.of(2017, 1, 1), null), - CDateRange.atLeast(LocalDate.of(2017, 1, 1)) - ) - , + "{\"min\":\"2017-01-01\"}", + new Range<>(LocalDate.of(2017, 1, 1), null), + CDateRange.atLeast(LocalDate.of(2017, 1, 1)) + ), Arguments.of( - "{\"max\":\"2017-01-01\"}", - new Range<>(null, LocalDate.of(2017, 1, 1)), - CDateRange.atMost(LocalDate.of(2017, 1, 1)) + "{\"max\":\"2017-01-01\"}", + new Range<>(null, LocalDate.of(2017, 1, 1)), + CDateRange.atMost(LocalDate.of(2017, 1, 1)) ) ); } - - @ParameterizedTest(name="{0}") @MethodSource + + @ParameterizedTest(name = "{0}") + @MethodSource public void deserialize(String json, Range expected, CDateRange expectedCDateRange) throws IOException { ObjectReader reader = Jackson.MAPPER.readerFor( Jackson.MAPPER.getTypeFactory().constructParametricType(Range.class, LocalDate.class) @@ -169,9 +195,9 @@ public void deserialize(String json, Range expected, CDateRange expec Range range = reader.readValue(json); assertThat(range).isEqualTo(expected); assertThat(range).isEqualToComparingFieldByFieldRecursively(expected); - + CDateRange cDateRange = CDateRange.of(range); assertThat(cDateRange).isEqualTo(expectedCDateRange); assertThat(cDateRange).isEqualToComparingFieldByFieldRecursively(expectedCDateRange); } -} \ No newline at end of file +} diff --git a/backend/src/test/java/com/bakdata/conquery/models/common/daterange/CDateRangeTest.java b/backend/src/test/java/com/bakdata/conquery/models/common/daterange/CDateRangeTest.java index 547f3e51af2..a01ee3cdd00 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/common/daterange/CDateRangeTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/common/daterange/CDateRangeTest.java @@ -48,4 +48,4 @@ public void spanClosed() { } -} \ No newline at end of file +} diff --git a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/filters/TestGroupFilter.java b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/filters/TestGroupFilter.java index 9300510f01c..9aef81f903f 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/filters/TestGroupFilter.java +++ b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/filters/TestGroupFilter.java @@ -1,5 +1,9 @@ package com.bakdata.conquery.models.datasets.concepts.filters; +import jakarta.validation.Valid; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; import java.util.ArrayList; import java.util.EnumSet; import java.util.List; @@ -7,10 +11,6 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.LongStream; -import jakarta.validation.Valid; -import jakarta.validation.constraints.Min; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.apiv1.frontend.FrontendFilterConfiguration; import com.bakdata.conquery.apiv1.frontend.FrontendFilterType; @@ -57,7 +57,11 @@ public boolean isValidColumnType() { final boolean acceptable = getAcceptedColumnTypes().contains(resolved.getType()); if (!acceptable) { - log.error("Column[{}] is of Type[{}]. Not one of [{}]", resolved.getId(), resolved.getType(), getAcceptedColumnTypes()); + log.error( + "Column[{}] is of Type[{}]. Not one of [{}]", + resolved.getId(), + resolved.getType(), + getAcceptedColumnTypes()); } return acceptable; @@ -83,14 +87,18 @@ public EventFilterNode createFilterNode(GroupFilterValue compoundFilterValue) private Map getFEFilter() { // TODO there is not yet a mismatch check between FEFilter and GroupedValues return Map.of( - GroupFilterValue.Fields.strings, FrontendFilterConfiguration.Nested.builder() - .label("Elements") - .type(FrontendFilterType.Fields.MULTI_SELECT) - .build(), - GroupFilterValue.Fields.repetitions, FrontendFilterConfiguration.Nested.builder() - .label("Maximum Repetitions") - .type(FrontendFilterType.Fields.INTEGER) - .build() + GroupFilterValue.Fields.strings, + FrontendFilterConfiguration.Nested.builder() + .label("Elements") + .type( + FrontendFilterType.Fields.MULTI_SELECT) + .build(), + GroupFilterValue.Fields.repetitions, + FrontendFilterConfiguration.Nested.builder() + .label("Maximum Repetitions") + .type( + FrontendFilterType.Fields.INTEGER) + .build() ); } @@ -118,10 +126,11 @@ public void resolve(QueryResolveContext context) { ArrayList values = new ArrayList<>(); for (String string : strings) { LongStream.range(1, repetitions + 1) - .mapToInt(Math::toIntExact) - .mapToObj(string::repeat) - .sequential() - .collect(Collectors.toCollection(() -> values)); + .mapToInt(Math::toIntExact) + .mapToObj( + string::repeat) + .sequential() + .collect(Collectors.toCollection(() -> values)); } resolvedValues = values.toArray(String[]::new); } diff --git a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/frontend/InternalFilterSearchItemTest.java b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/frontend/InternalFilterSearchItemTest.java index 8c174aa25cb..cda5c59113f 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/frontend/InternalFilterSearchItemTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/frontend/InternalFilterSearchItemTest.java @@ -27,7 +27,8 @@ public class InternalFilterSearchItemTest { @RegisterExtension private static final NamespaceStorageExtension NAMESPACE_STORAGE_EXTENSION = new NamespaceStorageExtension(); private static final NamespacedStorage NAMESPACED_STORAGE = NAMESPACE_STORAGE_EXTENSION.getStorage(); - public static final NamespacedStorageProvider STORAGE_PROVIDER = new TestNamespacedStorageProvider(NAMESPACED_STORAGE); + public static final NamespacedStorageProvider STORAGE_PROVIDER = new TestNamespacedStorageProvider( + NAMESPACED_STORAGE); @Test @@ -94,9 +95,9 @@ public void sortedValidityDates() throws Exception { FrontendTable feTable = new FrontEndConceptBuilder(new ConqueryConfig()).createTable(connector); assertThat(feTable.getDateColumn().getOptions()).containsExactly( - new FrontendValue(val0.getId().toString(), "val0"), - new FrontendValue(val1.getId().toString(), "val1"), - new FrontendValue(val2.getId().toString(), "val2") + new FrontendValue(val0.getId().toString(), "val0"), + new FrontendValue(val1.getId().toString(), "val1"), + new FrontendValue(val2.getId().toString(), "val2") ); } } diff --git a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/temporal/TemporalSamplerFactoryTest.java b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/temporal/TemporalSamplerFactoryTest.java index b0f6c90010e..de08f80186f 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/temporal/TemporalSamplerFactoryTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/temporal/TemporalSamplerFactoryTest.java @@ -18,17 +18,16 @@ class TemporalSamplerFactoryTest { public void earliest() { final Sampler sampler = EARLIEST.sampler(0); - assertThat(sampler.sample(generateSet(exactly(LocalDate.of(2011, 1, 10))))) - .hasValue(ofLocalDate(LocalDate.of(2011, 1, 10))); + assertThat(sampler.sample(generateSet(exactly(LocalDate.of(2011, 1, 10))))).hasValue( + ofLocalDate(LocalDate.of(2011, 1, 10))); - assertThat(sampler.sample(generateSet(atLeast(LocalDate.of(2011, 1, 10))))) - .hasValue(ofLocalDate(LocalDate.of(2011, 1, 10))); + assertThat(sampler.sample(generateSet(atLeast(LocalDate.of(2011, 1, 10))))).hasValue( + ofLocalDate(LocalDate.of(2011, 1, 10))); - assertThat(sampler.sample(generateSet(atMost(LocalDate.of(2011, 1, 10))))) - .isEmpty(); + assertThat(sampler.sample(generateSet(atMost(LocalDate.of(2011, 1, 10))))).isEmpty(); - assertThat(sampler.sample(generateSet(of(LocalDate.of(2011, 1, 10), LocalDate.of(2011, 1, 12))))) - .hasValue(ofLocalDate(LocalDate.of(2011, 1, 10))); + assertThat(sampler.sample(generateSet(of(LocalDate.of(2011, 1, 10), LocalDate.of(2011, 1, 12))))).hasValue( + ofLocalDate(LocalDate.of(2011, 1, 10))); } private static CDateSet generateSet(CDateRange... ranges) { @@ -39,23 +38,23 @@ private static CDateSet generateSet(CDateRange... ranges) { public void latest() { final Sampler sampler = LATEST.sampler(0); - assertThat(sampler.sample(generateSet(exactly(LocalDate.of(2011, 1, 10))))) - .hasValue(ofLocalDate(LocalDate.of(2011, 1, 10))); + assertThat(sampler.sample(generateSet(exactly(LocalDate.of(2011, 1, 10))))).hasValue( + ofLocalDate(LocalDate.of(2011, 1, 10))); - assertThat(sampler.sample(generateSet(atLeast(LocalDate.of(2011, 1, 10))))) - .hasValue(ofLocalDate(LocalDate.of(2011, 1, 10))); + assertThat(sampler.sample(generateSet(atLeast(LocalDate.of(2011, 1, 10))))).hasValue( + ofLocalDate(LocalDate.of(2011, 1, 10))); - assertThat(sampler.sample(generateSet(atMost(LocalDate.of(2011, 1, 10))))) - .hasValue(ofLocalDate(LocalDate.of(2011, 1, 10))); + assertThat(sampler.sample(generateSet(atMost(LocalDate.of(2011, 1, 10))))).hasValue( + ofLocalDate(LocalDate.of(2011, 1, 10))); - assertThat(sampler.sample(generateSet(of(LocalDate.of(2011, 1, 10), LocalDate.of(2011, 1, 12))))) - .hasValue(ofLocalDate(LocalDate.of(2011, 1, 12))); + assertThat(sampler.sample(generateSet(of(LocalDate.of(2011, 1, 10), LocalDate.of(2011, 1, 12))))).hasValue( + ofLocalDate(LocalDate.of(2011, 1, 12))); } @Test public void random() { - assertThat(RANDOM.sampler(0).sample(generateSet(exactly(LocalDate.of(2011, 1, 10))))) - .hasValue(ofLocalDate(LocalDate.of(2011, 1, 10))); + assertThat(RANDOM.sampler(0).sample(generateSet(exactly(LocalDate.of(2011, 1, 10))))).hasValue( + ofLocalDate(LocalDate.of(2011, 1, 10))); { assertThat(RANDOM.sampler(0).sample(CDateSet.createEmpty())).isEmpty(); @@ -66,25 +65,24 @@ public void random() { final Sampler sampler = RANDOM.sampler(0); for (int rep = 0; rep < 10; rep++) { - assertThat(sampler.sample(set).getAsInt()) - .matches(set::contains); + assertThat(sampler.sample(set).getAsInt()).matches(set::contains); } } { - final CDateSet set = - generateSet(of(LocalDate.of(2011, 1, 10), LocalDate.of(2011, 1, 12)), of(LocalDate.of(2011, 1, 15), LocalDate.of(2011, 1, 17))); + final CDateSet set = generateSet( + of(LocalDate.of(2011, 1, 10), LocalDate.of(2011, 1, 12)), + of(LocalDate.of(2011, 1, 15), LocalDate.of(2011, 1, 17))); final Sampler sampler = RANDOM.sampler(0); for (int rep = 0; rep < 10; rep++) { - assertThat(sampler.sample(set).getAsInt()) - .matches(set::contains); + assertThat(sampler.sample(set).getAsInt()).matches(set::contains); } } } -} \ No newline at end of file +} diff --git a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java index cf6a2965893..fad7270653d 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java @@ -2,13 +2,13 @@ import static org.assertj.core.api.Assertions.assertThat; +import jakarta.validation.Validator; import java.io.IOException; import java.util.Collections; import java.util.Map; import java.util.Random; import java.util.function.Supplier; import java.util.stream.Stream; -import jakarta.validation.Validator; import com.bakdata.conquery.io.jackson.Injectable; import com.bakdata.conquery.io.jackson.Jackson; @@ -36,7 +36,8 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -@Slf4j @Execution(ExecutionMode.SAME_THREAD) +@Slf4j +@Execution(ExecutionMode.SAME_THREAD) public class GroovyIndexedTest { public static final int SEED = 500; @@ -47,20 +48,70 @@ public class GroovyIndexedTest { public static Stream getTestKeys() { final Random random = new Random(SEED); Supplier>> rowMap = () -> new CalculatedValue<>( - () -> Collections.singletonMap("distinction", 8 + random.nextInt(10)) + () -> Collections.singletonMap("distinction", 8 + random.nextInt(10)) ); return Stream.of( - "A13B", "I43A", "H41B", "B05Z", "L02C", "L12Z", "H08A", "I56B", "I03A", "E79C", "B80Z", "I47A", "N13A", "G08B", "F43B", "P04A", "T36Z", "T36Z", "N11A", "D13A", "R01D", "F06A", "F24A", "O03Z", "P01Z", "R63D", "A13A", "O05A", "G29B", "I18A", "J08A", "E74Z", "D06C", "H36Z", "H05Z", "P65B", "I09A", "A66Z", "F12E", "Q60E", "I46B", "I97Z", "I78Z", "T01B", "J24C", "A62Z", "Q01Z", "N25Z", "A01B", "G02A" - , "ZULU" // This may not fail, but return null on both sides - ) - .map(v -> Arguments.of(v, rowMap.get())); + "A13B", + "I43A", + "H41B", + "B05Z", + "L02C", + "L12Z", + "H08A", + "I56B", + "I03A", + "E79C", + "B80Z", + "I47A", + "N13A", + "G08B", + "F43B", + "P04A", + "T36Z", + "T36Z", + "N11A", + "D13A", + "R01D", + "F06A", + "F24A", + "O03Z", + "P01Z", + "R63D", + "A13A", + "O05A", + "G29B", + "I18A", + "J08A", + "E74Z", + "D06C", + "H36Z", + "H05Z", + "P65B", + "I09A", + "A66Z", + "F12E", + "Q60E", + "I46B", + "I97Z", + "I78Z", + "T01B", + "J24C", + "A62Z", + "Q01Z", + "N25Z", + "A01B", + "G02A", + "ZULU" // This may not fail, but return null on both sides + ).map(v -> Arguments.of(v, rowMap.get())); } @BeforeAll public static void init() throws IOException, JSONException, ConfigurationException { final ObjectMapper mapper = Jackson.copyMapperAndInjectables(Jackson.MAPPER); - ObjectNode node = mapper.readerFor(ObjectNode.class).readValue(GroovyIndexedTest.class.getResourceAsStream(CONCEPT_SOURCE)); + ObjectNode node = mapper.readerFor(ObjectNode.class) + .readValue( + GroovyIndexedTest.class.getResourceAsStream(CONCEPT_SOURCE)); // load concept tree from json final NamespaceStorage storage = new NonPersistentStoreFactory().createNamespaceStorage(); @@ -91,7 +142,8 @@ public static void init() throws IOException, JSONException, ConfigurationExcept // Prepare Serdes injections final Validator validator = Validators.newValidator(); - final ObjectReader conceptReader = ((Injectable) values -> values.add(Validator.class, validator)).injectInto(mapper).readerFor(Concept.class); + final ObjectReader conceptReader = ((Injectable) values -> values.add(Validator.class, validator)).injectInto( + mapper).readerFor(Concept.class); // load tree twice to to avoid references indexedConcept = conceptReader.readValue(node); @@ -118,4 +170,4 @@ public void basic(String key, CalculatedValue> rowMap) throw } -} \ No newline at end of file +} diff --git a/backend/src/test/java/com/bakdata/conquery/models/error/ConqueryErrorTest.java b/backend/src/test/java/com/bakdata/conquery/models/error/ConqueryErrorTest.java index c160e773597..6e7ecd43f13 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/error/ConqueryErrorTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/error/ConqueryErrorTest.java @@ -13,12 +13,14 @@ public class ConqueryErrorTest { @Test public void errorConvertion() { final ExternalResolveFormatError error = new ExternalResolveFormatError(5, 6); - assertThat(error.asPlain()).isEqualTo(new SimpleErrorInfo(error.getId(), "CQ_EXECUTION_CREATION_RESOLVE_EXTERNAL_FORMAT", error.getMessage())); + assertThat(error.asPlain()).isEqualTo( + new SimpleErrorInfo(error.getId(), "CQ_EXECUTION_CREATION_RESOLVE_EXTERNAL_FORMAT", error.getMessage())); } @Test public void errorDeserialization() throws JsonProcessingException { - final SimpleErrorInfo error = Jackson.MAPPER.readerFor(SimpleErrorInfo.class).readValue( + final SimpleErrorInfo error = Jackson.MAPPER.readerFor(SimpleErrorInfo.class) + .readValue( """ {\r "code": "TEST_ERROR",\r @@ -26,7 +28,11 @@ public void errorDeserialization() throws JsonProcessingException { "message": "group was empty."\r }"""); - assertThat(error).isEqualTo(new SimpleErrorInfo(UUID.fromString("c8be5f10-1ea8-11eb-8fb8-26885ec43e14"), "TEST_ERROR", "group was empty.")); + assertThat(error).isEqualTo( + new SimpleErrorInfo( + UUID.fromString("c8be5f10-1ea8-11eb-8fb8-26885ec43e14"), + "TEST_ERROR", + "group was empty.")); } } diff --git a/backend/src/test/java/com/bakdata/conquery/models/events/stores/primitive/BooleanStoreTest.java b/backend/src/test/java/com/bakdata/conquery/models/events/stores/primitive/BooleanStoreTest.java index 7d4a8bd3a7e..b91ed85a59c 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/events/stores/primitive/BooleanStoreTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/events/stores/primitive/BooleanStoreTest.java @@ -18,8 +18,7 @@ public void select() { final BitSetStore booleanStore = new BitSetStore(values, new BitSet(10), 10); - assertThat(booleanStore.select(new int[]{0}, new int[]{10}).getValues()) - .isEqualTo(values); + assertThat(booleanStore.select(new int[]{0}, new int[]{10}).getValues()).isEqualTo(values); { final BitSet expected = new BitSet(); @@ -27,16 +26,14 @@ public void select() { expected.set(1); expected.set(2, false); - assertThat(booleanStore.select(new int[]{0, 6}, new int[]{1, 2}).getValues()) - .isEqualTo(expected); + assertThat(booleanStore.select(new int[]{0, 6}, new int[]{1, 2}).getValues()).isEqualTo(expected); } { final BitSet expected = new BitSet(); expected.set(0); - assertThat(booleanStore.select(new int[]{0}, new int[]{1}).getValues()) - .isEqualTo(expected); + assertThat(booleanStore.select(new int[]{0}, new int[]{1}).getValues()).isEqualTo(expected); } } @@ -51,7 +48,9 @@ public void testSerialization() throws JsonProcessingException { final BitSetStore booleanStore = new BitSetStore(bitSet, new BitSet(128), 128); - final BitSetStore booleanStore1 = Jackson.MAPPER.readValue(Jackson.MAPPER.writeValueAsString(booleanStore), BitSetStore.class); + final BitSetStore booleanStore1 = Jackson.MAPPER.readValue( + Jackson.MAPPER.writeValueAsString(booleanStore), + BitSetStore.class); assertThat(booleanStore1.getValues().get(128)).isEqualTo(false); @@ -75,4 +74,4 @@ public void danglingFalse() { assertThat(store.getBoolean(9)).isFalse(); } -} \ No newline at end of file +} diff --git a/backend/src/test/java/com/bakdata/conquery/models/events/stores/primitive/IntArrayStoreTest.java b/backend/src/test/java/com/bakdata/conquery/models/events/stores/primitive/IntArrayStoreTest.java index 631a85bbf3f..c7d92643166 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/events/stores/primitive/IntArrayStoreTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/events/stores/primitive/IntArrayStoreTest.java @@ -17,7 +17,7 @@ public void integerStore() { final IntArrayStore store = IntArrayStore.create(values.size()); for (int index = 0; index < values.size(); index++) { - if(values.get(index) == null){ + if (values.get(index) == null) { store.setNull(index); continue; } @@ -27,8 +27,7 @@ public void integerStore() { for (int index = 0; index < values.size(); index++) { if (values.get(index) == null) { assertThat(store.has(index)).isFalse(); - } - else { + } else { assertThat(store.getInteger(index)).isEqualTo(values.get(index)); } } @@ -48,7 +47,7 @@ public void byteStore() { final ByteArrayStore store = ByteArrayStore.create(values.size()); for (int index = 0; index < values.size(); index++) { - if(values.get(index) == null){ + if (values.get(index) == null) { store.setNull(index); continue; } @@ -59,8 +58,7 @@ public void byteStore() { for (int index = 0; index < values.size(); index++) { if (values.get(index) == null) { assertThat(store.has(index)).isFalse(); - } - else { + } else { assertThat(store.getInteger(index)).isEqualTo(values.get(index)); } } @@ -82,7 +80,7 @@ public void shortStore() { final ShortArrayStore store = ShortArrayStore.create(values.size()); for (int index = 0; index < values.size(); index++) { - if(values.get(index) == null){ + if (values.get(index) == null) { store.setNull(index); continue; } @@ -93,8 +91,7 @@ public void shortStore() { for (int index = 0; index < values.size(); index++) { if (values.get(index) == null) { assertThat(store.has(index)).isFalse(); - } - else { + } else { assertThat(store.getInteger(index)).isEqualTo(values.get(index)); } } @@ -109,4 +106,4 @@ public void shortStore() { } -} \ No newline at end of file +} diff --git a/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/ColumnStoreSerializationTests.java b/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/ColumnStoreSerializationTests.java index 8992c4c8561..1549d9e4a6a 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/ColumnStoreSerializationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/ColumnStoreSerializationTests.java @@ -52,7 +52,9 @@ public class ColumnStoreSerializationTests { */ private static final Set> EXCLUDING = Set.of(CompoundDateRangeStore.class); - private static final NamespaceStorage STORAGE = new NamespaceStorage(new NonPersistentStoreFactory(), "ColumnStoreSerializationTests"); + private static final NamespaceStorage STORAGE = new NamespaceStorage( + new NonPersistentStoreFactory(), + "ColumnStoreSerializationTests"); private static ObjectMapper shardInternalMapper; private static ConqueryConfig config; @@ -73,40 +75,35 @@ public static void setupRegistry() { @SuppressWarnings({"unchecked", "rawtypes"}) public void testAllTypesCovered() { assertThat( - createCTypes() - .stream() - .map(Object::getClass) - .collect(Collectors.toSet()) - - ) - .containsAll( - Sets.difference( - (Set) CPSTypeIdResolver.listImplementations(ColumnStore.class), - EXCLUDING - ) - ) - .doesNotContainAnyElementsOf(EXCLUDING); + createCTypes().stream().map(Object::getClass).collect(Collectors.toSet()) + + ).containsAll( + Sets.difference( + (Set) CPSTypeIdResolver.listImplementations(ColumnStore.class), + EXCLUDING + ) + ).doesNotContainAnyElementsOf(EXCLUDING); } public static List createCTypes() { return Arrays.asList( - new ScaledDecimalStore(13, IntArrayStore.create(10)), - new MoneyIntStore(IntArrayStore.create(10), 2).config(config), - new DirectDateRangeStore(IntegerDateStore.create(10), IntegerDateStore.create(10)), - new QuarterDateRangeStore(LongArrayStore.create(10)), - new IntegerDateStore(LongArrayStore.create(10)), - StringStoreString.withInternedStrings(new String[]{"a", "b", "c"}), - DecimalArrayStore.create(10), - LongArrayStore.create(10), - IntArrayStore.create(10), - ByteArrayStore.create(10), - ShortArrayStore.create(10), - FloatArrayStore.create(10), - DoubleArrayStore.create(10), - BitSetStore.create(10), - EmptyStore.INSTANCE, - new RebasingIntegerStore(10, 10, IntArrayStore.create(10)) + new ScaledDecimalStore(13, IntArrayStore.create(10)), + new MoneyIntStore(IntArrayStore.create(10), 2).config(config), + new DirectDateRangeStore(IntegerDateStore.create(10), IntegerDateStore.create(10)), + new QuarterDateRangeStore(LongArrayStore.create(10)), + new IntegerDateStore(LongArrayStore.create(10)), + StringStoreString.withInternedStrings(new String[]{"a", "b", "c"}), + DecimalArrayStore.create(10), + LongArrayStore.create(10), + IntArrayStore.create(10), + ByteArrayStore.create(10), + ShortArrayStore.create(10), + FloatArrayStore.create(10), + DoubleArrayStore.create(10), + BitSetStore.create(10), + EmptyStore.INSTANCE, + new RebasingIntegerStore(10, 10, IntArrayStore.create(10)) ); } @@ -114,9 +111,6 @@ public static List createCTypes() { @MethodSource("createCTypes") public void testSerialization(ColumnStore type) throws IOException, JSONException { - SerializationTestUtil - .forType(ColumnStore.class) - .objectMappers(shardInternalMapper) - .test(type); + SerializationTestUtil.forType(ColumnStore.class).objectMappers(shardInternalMapper).test(type); } } diff --git a/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/MajorTypesTest.java b/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/MajorTypesTest.java index e62584181b2..d3d4d68ba81 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/MajorTypesTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/MajorTypesTest.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.models.events.stores.types; import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.config.ParserConfig; import com.bakdata.conquery.models.events.MajorTypeId; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; diff --git a/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java b/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java index 98ee9f74161..29161fbb185 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java @@ -38,8 +38,9 @@ public class DefaultLabelTest { public static final ConqueryConfig CONFIG = new ConqueryConfig(); private final static MetaStorage META_STORAGE = new NonPersistentStoreFactory().createMetaStorage(); private final static NamespaceStorage NAMESPACE_STORAGE = new NonPersistentStoreFactory().createNamespaceStorage(); - public static final NamespacedStorageProvider STORAGE_PROVIDER = new TestNamespacedStorageProvider(NAMESPACE_STORAGE); - private static final Dataset DATASET = new Dataset("dataset"); + public static final NamespacedStorageProvider STORAGE_PROVIDER = new TestNamespacedStorageProvider( + NAMESPACE_STORAGE); + private static final Dataset DATASET = new Dataset("dataset"); private static final User user = new User("user", "user", META_STORAGE); private static final TreeConcept CONCEPT = new TreeConcept(); @@ -65,9 +66,7 @@ public static void beforeAll() throws Exception { } @ParameterizedTest - @CsvSource({ - "de,Concept", - "en,Concept" + @CsvSource({"de,Concept", "en,Concept" }) void autoLabelConceptQuery(Locale locale, String autoLabel) { I18n.LOCALE.set(locale); @@ -96,9 +95,7 @@ private PrintSettings getPrintSettings(Locale locale) { } @ParameterizedTest - @CsvSource({ - "de,Default-Concept", - "en,Default-Concept" + @CsvSource({"de,Default-Concept", "en,Default-Concept" }) void autoLabelConceptQueryFallback(Locale locale, String autoLabel) { I18n.LOCALE.set(locale); @@ -120,14 +117,18 @@ void autoLabelConceptQueryFallback(Locale locale, String autoLabel) { @ParameterizedTest - @CsvSource({ - "de,Anfrage", - "en,Query" + @CsvSource({"de,Anfrage", "en,Query" }) void autoLabelReusedQuery(Locale locale, String autoLabel) { I18n.LOCALE.set(locale); - final ManagedQuery managedQuery = new ManagedQuery(null, new UserId("test"), DATASET.getId(), META_STORAGE, null, CONFIG); + final ManagedQuery managedQuery = new ManagedQuery( + null, + new UserId("test"), + DATASET.getId(), + META_STORAGE, + null, + CONFIG); managedQuery.setQueryId(UUID.randomUUID()); CQReusedQuery reused = new CQReusedQuery(managedQuery.getId()); @@ -142,9 +143,7 @@ void autoLabelReusedQuery(Locale locale, String autoLabel) { @ParameterizedTest - @CsvSource({ - "de,Hochgeladene-Liste", - "en,Uploaded-List" + @CsvSource({"de,Hochgeladene-Liste", "en,Uploaded-List" }) void autoLabelUploadQuery(Locale locale, String autoLabel) { I18n.LOCALE.set(locale); @@ -160,14 +159,18 @@ void autoLabelUploadQuery(Locale locale, String autoLabel) { } @ParameterizedTest - @CsvSource({ - "de,Hochgeladene-Liste Anfrage Concept1 Concept2 und weitere", - "en,Uploaded-List Query Concept1 Concept2 and further" + @CsvSource({"de,Hochgeladene-Liste Anfrage Concept1 Concept2 und weitere", "en,Uploaded-List Query Concept1 Concept2 and further" }) void autoLabelComplexQuery(Locale locale, String autoLabel) { I18n.LOCALE.set(locale); - final ManagedQuery managedQuery = new ManagedQuery(null, new UserId("test"), DATASET.getId(), META_STORAGE, null, CONFIG); + final ManagedQuery managedQuery = new ManagedQuery( + null, + new UserId("test"), + DATASET.getId(), + META_STORAGE, + null, + CONFIG); managedQuery.setMetaStorage(META_STORAGE); managedQuery.setQueryId(UUID.randomUUID()); @@ -175,15 +178,17 @@ void autoLabelComplexQuery(Locale locale, String autoLabel) { CQAnd and = new CQAnd(); CQConcept concept1 = makeCQConceptWithLabel("Concept1"); CQConcept concept2 = makeCQConceptWithLabel("Concept2"); - CQConcept concept3 = makeCQConceptWithLabel("Concept3veryveryveryveryveryveryveryverylooooooooooooooooooooonglabel"); + CQConcept concept3 = makeCQConceptWithLabel( + "Concept3veryveryveryveryveryveryveryverylooooooooooooooooooooonglabel"); - and.setChildren(List.of( + and.setChildren( + List.of( new CQExternal(List.of(), new String[0][0], false), new CQReusedQuery(managedQuery.getId()), concept1, concept2, concept3 - )); + )); ConceptQuery cq = new ConceptQuery(and); ManagedQuery mQuery = cq.toManagedExecution(user.getId(), DATASET.getId(), META_STORAGE, null, CONFIG); @@ -195,14 +200,18 @@ void autoLabelComplexQuery(Locale locale, String autoLabel) { @ParameterizedTest - @CsvSource({ - "de,Hochgeladene-Liste Anfrage Default-Concept Concept2 Concept3", - "en,Uploaded-List Query Default-Concept Concept2 Concept3" + @CsvSource({"de,Hochgeladene-Liste Anfrage Default-Concept Concept2 Concept3", "en,Uploaded-List Query Default-Concept Concept2 Concept3" }) void autoLabelComplexQueryNullLabels(Locale locale, String autoLabel) { I18n.LOCALE.set(locale); - final ManagedQuery managedQuery = new ManagedQuery(null, new UserId("test"), DATASET.getId(), META_STORAGE, null, CONFIG); + final ManagedQuery managedQuery = new ManagedQuery( + null, + new UserId("test"), + DATASET.getId(), + META_STORAGE, + null, + CONFIG); managedQuery.setQueryId(UUID.randomUUID()); CQAnd and = new CQAnd(); @@ -211,13 +220,14 @@ void autoLabelComplexQueryNullLabels(Locale locale, String autoLabel) { concept1.setElements(List.of(CONCEPT.getId())); CQConcept concept2 = makeCQConceptWithLabel("Concept2"); CQConcept concept3 = makeCQConceptWithLabel("Concept3"); - and.setChildren(List.of( + and.setChildren( + List.of( new CQExternal(List.of(), new String[0][0], false), new CQReusedQuery(managedQuery.getId()), concept1, concept2, concept3 - )); + )); ConceptQuery cq = new ConceptQuery(and); ManagedQuery mQuery = cq.toManagedExecution(user.getId(), DATASET.getId(), META_STORAGE, null, CONFIG); @@ -228,10 +238,10 @@ void autoLabelComplexQueryNullLabels(Locale locale, String autoLabel) { } @ParameterizedTest - @CsvSource(value = { - "de;Datenexport", - "en;Data Export" - }, delimiter = ';') + @CsvSource( + value = {"de;Datenexport", "en;Data Export" + }, + delimiter = ';') void autoLabelExportForm(Locale locale, String autoLabel) { I18n.LOCALE.set(locale); diff --git a/backend/src/test/java/com/bakdata/conquery/models/forms/DateContextTest.java b/backend/src/test/java/com/bakdata/conquery/models/forms/DateContextTest.java index 758c08db78d..8f5554efcdb 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/forms/DateContextTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/forms/DateContextTest.java @@ -23,7 +23,9 @@ public class DateContextTest { public void rangeAbsCompleteTestWithCoarse() { // Because COMPLETE_ONLY is the most coarse subdivision mode this has the same output as DateContextTest#rangeAbsCompleteTestWithoutCoarse CDateRange mask = CDateRange.of(LocalDate.of(2001, 5, 23), LocalDate.of(2005, 4, 21)); - List contexts = DateContext.generateAbsoluteContexts(mask, List.of(ExportForm.ResolutionAndAlignment.of(COMPLETE, NO_ALIGN))); + List contexts = DateContext.generateAbsoluteContexts( + mask, + List.of(ExportForm.ResolutionAndAlignment.of(COMPLETE, NO_ALIGN))); assertThat(contexts).containsExactly( new DateContext(mask, FeatureGroup.SINGLE_GROUP, null, null, COMPLETE) @@ -33,40 +35,49 @@ public void rangeAbsCompleteTestWithCoarse() { @Test public void rangeAbsYearTestWithCoarse() { CDateRange mask = CDateRange.of(LocalDate.of(2001, 5, 23), LocalDate.of(2005, 4, 21)); - List contexts = DateContext.generateAbsoluteContexts(mask, ExportForm.getResolutionAlignmentMap(YEARS.getThisAndCoarserSubdivisions(),YEAR)); + List contexts = DateContext.generateAbsoluteContexts( + mask, + ExportForm.getResolutionAlignmentMap(YEARS.getThisAndCoarserSubdivisions(), YEAR)); - assertThat(contexts).extracting(DateContext::getDateRange).containsExactly ( + assertThat(contexts).extracting(DateContext::getDateRange) + .containsExactly( mask, CDateRange.of(LocalDate.of(2001, 5, 23), LocalDate.of(2001, 12, 31)), CDateRange.of(LocalDate.of(2002, 1, 1), LocalDate.of(2002, 12, 31)), CDateRange.of(LocalDate.of(2003, 1, 1), LocalDate.of(2003, 12, 31)), CDateRange.of(LocalDate.of(2004, 1, 1), LocalDate.of(2004, 12, 31)), CDateRange.of(LocalDate.of(2005, 1, 1), LocalDate.of(2005, 4, 21)) - ); + ); assertThat(contexts).extracting(DateContext::getFeatureGroup).containsOnly(FeatureGroup.SINGLE_GROUP); } - + @Test public void rangeAbsYearTestWithoutCoarse() { CDateRange mask = CDateRange.of(LocalDate.of(2001, 5, 23), LocalDate.of(2005, 4, 21)); - List contexts = DateContext.generateAbsoluteContexts(mask, ExportForm.getResolutionAlignmentMap(List.of(YEARS),YEAR)); + List contexts = DateContext.generateAbsoluteContexts( + mask, + ExportForm.getResolutionAlignmentMap(List.of(YEARS), YEAR)); - assertThat(contexts).extracting(DateContext::getDateRange).containsExactly ( + assertThat(contexts).extracting(DateContext::getDateRange) + .containsExactly( CDateRange.of(LocalDate.of(2001, 5, 23), LocalDate.of(2001, 12, 31)), CDateRange.of(LocalDate.of(2002, 1, 1), LocalDate.of(2002, 12, 31)), CDateRange.of(LocalDate.of(2003, 1, 1), LocalDate.of(2003, 12, 31)), CDateRange.of(LocalDate.of(2004, 1, 1), LocalDate.of(2004, 12, 31)), CDateRange.of(LocalDate.of(2005, 1, 1), LocalDate.of(2005, 4, 21)) - ); + ); assertThat(contexts).extracting(DateContext::getFeatureGroup).containsOnly(FeatureGroup.SINGLE_GROUP); } @Test public void rangeAbsQuarterTestWithCoarse() { CDateRange mask = CDateRange.of(LocalDate.of(2001, 5, 23), LocalDate.of(2003, 4, 21)); - List contexts = DateContext.generateAbsoluteContexts(mask, ExportForm.getResolutionAlignmentMap(QUARTERS.getThisAndCoarserSubdivisions(), QUARTER)); + List contexts = DateContext.generateAbsoluteContexts( + mask, + ExportForm.getResolutionAlignmentMap(QUARTERS.getThisAndCoarserSubdivisions(), QUARTER)); - assertThat(contexts).extracting(DateContext::getDateRange).containsExactly ( + assertThat(contexts).extracting(DateContext::getDateRange) + .containsExactly( // Complete mask, // Years @@ -83,16 +94,19 @@ public void rangeAbsQuarterTestWithCoarse() { CDateRange.of(LocalDate.of(2002, 10, 1), LocalDate.of(2002, 12, 31)), CDateRange.of(LocalDate.of(2003, 1, 1), LocalDate.of(2003, 3, 31)), CDateRange.of(LocalDate.of(2003, 4, 1), LocalDate.of(2003, 4, 21)) - ); + ); assertThat(contexts).extracting(DateContext::getFeatureGroup).containsOnly(FeatureGroup.SINGLE_GROUP); } - + @Test public void rangeAbsQuarterTestWithoutCoarse() { CDateRange mask = CDateRange.of(LocalDate.of(2001, 5, 23), LocalDate.of(2003, 4, 21)); - List contexts = DateContext.generateAbsoluteContexts(mask, List.of(ExportForm.ResolutionAndAlignment.of(QUARTERS, QUARTER))); + List contexts = DateContext.generateAbsoluteContexts( + mask, + List.of(ExportForm.ResolutionAndAlignment.of(QUARTERS, QUARTER))); - assertThat(contexts).extracting(DateContext::getDateRange).containsExactly ( + assertThat(contexts).extracting(DateContext::getDateRange) + .containsExactly( CDateRange.of(LocalDate.of(2001, 5, 23), LocalDate.of(2001, 6, 30)), CDateRange.of(LocalDate.of(2001, 7, 1), LocalDate.of(2001, 9, 30)), CDateRange.of(LocalDate.of(2001, 10, 1), LocalDate.of(2001, 12, 31)), @@ -102,7 +116,7 @@ public void rangeAbsQuarterTestWithoutCoarse() { CDateRange.of(LocalDate.of(2002, 10, 1), LocalDate.of(2002, 12, 31)), CDateRange.of(LocalDate.of(2003, 1, 1), LocalDate.of(2003, 3, 31)), CDateRange.of(LocalDate.of(2003, 4, 1), LocalDate.of(2003, 4, 21)) - ); + ); assertThat(contexts).extracting(DateContext::getFeatureGroup).containsOnly(FeatureGroup.SINGLE_GROUP); } @@ -116,14 +130,47 @@ public void rangeRelDaysBeforeTest() { IndexPlacement indexPlacement = IndexPlacement.BEFORE; - List contexts = DateContext.generateRelativeContexts(event, indexPlacement, featureTime, outcomeTime, timeUnit, List.of(ExportForm.ResolutionAndAlignment.of(COMPLETE, NO_ALIGN), ExportForm.ResolutionAndAlignment.of(DAYS, DAY))); + List contexts = DateContext.generateRelativeContexts( + event, + indexPlacement, + featureTime, + outcomeTime, + timeUnit, + List.of( + ExportForm.ResolutionAndAlignment.of(COMPLETE, NO_ALIGN), + ExportForm.ResolutionAndAlignment.of(DAYS, DAY))); - assertThat(contexts).containsExactly ( - new DateContext(CDateRange.of(LocalDate.of(2001, 5, 22), LocalDate.of(2001, 5, 23)), FeatureGroup.FEATURE, null, eventDate, COMPLETE), - new DateContext(CDateRange.of(LocalDate.of(2001, 5, 24), LocalDate.of(2001, 5, 24)), FeatureGroup.OUTCOME, null, eventDate, COMPLETE), - new DateContext(CDateRange.of(LocalDate.of(2001, 5, 22), LocalDate.of(2001, 5, 22)), FeatureGroup.FEATURE, -2, eventDate, DAYS), - new DateContext(CDateRange.of(LocalDate.of(2001, 5, 23), LocalDate.of(2001, 5, 23)), FeatureGroup.FEATURE, -1, eventDate, DAYS), - new DateContext(CDateRange.of(LocalDate.of(2001, 5, 24), LocalDate.of(2001, 5, 24)), FeatureGroup.OUTCOME, 1, eventDate, DAYS) + assertThat(contexts).containsExactly( + new DateContext( + CDateRange.of(LocalDate.of(2001, 5, 22), LocalDate.of(2001, 5, 23)), + FeatureGroup.FEATURE, + null, + eventDate, + COMPLETE), + new DateContext( + CDateRange.of(LocalDate.of(2001, 5, 24), LocalDate.of(2001, 5, 24)), + FeatureGroup.OUTCOME, + null, + eventDate, + COMPLETE), + new DateContext( + CDateRange.of(LocalDate.of(2001, 5, 22), LocalDate.of(2001, 5, 22)), + FeatureGroup.FEATURE, + -2, + eventDate, + DAYS), + new DateContext( + CDateRange.of(LocalDate.of(2001, 5, 23), LocalDate.of(2001, 5, 23)), + FeatureGroup.FEATURE, + -1, + eventDate, + DAYS), + new DateContext( + CDateRange.of(LocalDate.of(2001, 5, 24), LocalDate.of(2001, 5, 24)), + FeatureGroup.OUTCOME, + 1, + eventDate, + DAYS) ); } @@ -137,11 +184,27 @@ public void rangeRelDaysBeforeCompleteOnlyTest() { IndexPlacement indexPlacement = IndexPlacement.BEFORE; - List contexts = DateContext.generateRelativeContexts(event, indexPlacement, featureTime, outcomeTime, timeUnit, List.of(ExportForm.ResolutionAndAlignment.of(COMPLETE, NO_ALIGN))); + List contexts = DateContext.generateRelativeContexts( + event, + indexPlacement, + featureTime, + outcomeTime, + timeUnit, + List.of(ExportForm.ResolutionAndAlignment.of(COMPLETE, NO_ALIGN))); List expectedRanges = Arrays.asList( - new DateContext(CDateRange.of(LocalDate.of(2001, 5, 22), LocalDate.of(2001, 5, 23)), FeatureGroup.FEATURE, null, eventDate, COMPLETE), - new DateContext(CDateRange.of(LocalDate.of(2001, 5, 24), LocalDate.of(2001, 5, 24)), FeatureGroup.OUTCOME, null, eventDate, COMPLETE) + new DateContext( + CDateRange.of(LocalDate.of(2001, 5, 22), LocalDate.of(2001, 5, 23)), + FeatureGroup.FEATURE, + null, + eventDate, + COMPLETE), + new DateContext( + CDateRange.of(LocalDate.of(2001, 5, 24), LocalDate.of(2001, 5, 24)), + FeatureGroup.OUTCOME, + null, + eventDate, + COMPLETE) ); assertThat(contexts).containsExactlyElementsOf(expectedRanges); } @@ -156,17 +219,51 @@ public void rangeRelDaysAfterTest() { IndexPlacement indexPlacement = IndexPlacement.AFTER; - List - contexts = - DateContext.generateRelativeContexts(event, indexPlacement, featureTime, outcomeTime, timeUnit, ExportForm.getResolutionAlignmentMap(List.of(COMPLETE, DAYS), DAY)); + List contexts = DateContext.generateRelativeContexts( + event, + indexPlacement, + featureTime, + outcomeTime, + timeUnit, + ExportForm.getResolutionAlignmentMap(List.of(COMPLETE, DAYS), DAY)); assertThat(contexts).containsExactly( - new DateContext(CDateRange.of(LocalDate.of(2001, 5, 21), LocalDate.of(2001, 5, 22)), FeatureGroup.FEATURE, null, eventDate, COMPLETE), - new DateContext(CDateRange.of(LocalDate.of(2001, 5, 23), LocalDate.of(2001, 5, 24)), FeatureGroup.OUTCOME, null, eventDate, COMPLETE), - new DateContext(CDateRange.of(LocalDate.of(2001, 5, 21), LocalDate.of(2001, 5, 21)), FeatureGroup.FEATURE, -2, eventDate, DAYS), - new DateContext(CDateRange.of(LocalDate.of(2001, 5, 22), LocalDate.of(2001, 5, 22)), FeatureGroup.FEATURE, -1, eventDate, DAYS), - new DateContext(CDateRange.of(LocalDate.of(2001, 5, 23), LocalDate.of(2001, 5, 23)), FeatureGroup.OUTCOME, 1, eventDate, DAYS), - new DateContext(CDateRange.of(LocalDate.of(2001, 5, 24), LocalDate.of(2001, 5, 24)), FeatureGroup.OUTCOME, 2, eventDate, DAYS) + new DateContext( + CDateRange.of(LocalDate.of(2001, 5, 21), LocalDate.of(2001, 5, 22)), + FeatureGroup.FEATURE, + null, + eventDate, + COMPLETE), + new DateContext( + CDateRange.of(LocalDate.of(2001, 5, 23), LocalDate.of(2001, 5, 24)), + FeatureGroup.OUTCOME, + null, + eventDate, + COMPLETE), + new DateContext( + CDateRange.of(LocalDate.of(2001, 5, 21), LocalDate.of(2001, 5, 21)), + FeatureGroup.FEATURE, + -2, + eventDate, + DAYS), + new DateContext( + CDateRange.of(LocalDate.of(2001, 5, 22), LocalDate.of(2001, 5, 22)), + FeatureGroup.FEATURE, + -1, + eventDate, + DAYS), + new DateContext( + CDateRange.of(LocalDate.of(2001, 5, 23), LocalDate.of(2001, 5, 23)), + FeatureGroup.OUTCOME, + 1, + eventDate, + DAYS), + new DateContext( + CDateRange.of(LocalDate.of(2001, 5, 24), LocalDate.of(2001, 5, 24)), + FeatureGroup.OUTCOME, + 2, + eventDate, + DAYS) ); } @@ -180,15 +277,51 @@ public void rangeRelDaysNeutralTest() { IndexPlacement indexPlacement = IndexPlacement.NEUTRAL; - List contexts = DateContext.generateRelativeContexts(event, indexPlacement, featureTime, outcomeTime, timeUnit, ExportForm.getResolutionAlignmentMap(List.of(COMPLETE,DAYS), DAY)); + List contexts = DateContext.generateRelativeContexts( + event, + indexPlacement, + featureTime, + outcomeTime, + timeUnit, + ExportForm.getResolutionAlignmentMap(List.of(COMPLETE, DAYS), DAY)); - assertThat(contexts).containsExactly ( - new DateContext(CDateRange.of(LocalDate.of(2001, 5, 21), LocalDate.of(2001, 5, 22)), FeatureGroup.FEATURE, null, eventDate, COMPLETE), - new DateContext(CDateRange.of(LocalDate.of(2001, 5, 24), LocalDate.of(2001, 5, 25)), FeatureGroup.OUTCOME, null, eventDate, COMPLETE), - new DateContext(CDateRange.of(LocalDate.of(2001, 5, 21), LocalDate.of(2001, 5, 21)), FeatureGroup.FEATURE, -2, eventDate, DAYS), - new DateContext(CDateRange.of(LocalDate.of(2001, 5, 22), LocalDate.of(2001, 5, 22)), FeatureGroup.FEATURE, -1, eventDate, DAYS), - new DateContext(CDateRange.of(LocalDate.of(2001, 5, 24), LocalDate.of(2001, 5, 24)), FeatureGroup.OUTCOME, 1, eventDate, DAYS), - new DateContext(CDateRange.of(LocalDate.of(2001, 5, 25), LocalDate.of(2001, 5, 25)), FeatureGroup.OUTCOME, 2, eventDate, DAYS) + assertThat(contexts).containsExactly( + new DateContext( + CDateRange.of(LocalDate.of(2001, 5, 21), LocalDate.of(2001, 5, 22)), + FeatureGroup.FEATURE, + null, + eventDate, + COMPLETE), + new DateContext( + CDateRange.of(LocalDate.of(2001, 5, 24), LocalDate.of(2001, 5, 25)), + FeatureGroup.OUTCOME, + null, + eventDate, + COMPLETE), + new DateContext( + CDateRange.of(LocalDate.of(2001, 5, 21), LocalDate.of(2001, 5, 21)), + FeatureGroup.FEATURE, + -2, + eventDate, + DAYS), + new DateContext( + CDateRange.of(LocalDate.of(2001, 5, 22), LocalDate.of(2001, 5, 22)), + FeatureGroup.FEATURE, + -1, + eventDate, + DAYS), + new DateContext( + CDateRange.of(LocalDate.of(2001, 5, 24), LocalDate.of(2001, 5, 24)), + FeatureGroup.OUTCOME, + 1, + eventDate, + DAYS), + new DateContext( + CDateRange.of(LocalDate.of(2001, 5, 25), LocalDate.of(2001, 5, 25)), + FeatureGroup.OUTCOME, + 2, + eventDate, + DAYS) ); } @@ -202,14 +335,45 @@ public void rangeRelQuarterBeforeTest() { IndexPlacement indexPlacement = IndexPlacement.BEFORE; - List contexts = DateContext.generateRelativeContexts(event, indexPlacement, featureTime, outcomeTime, timeUnit, ExportForm.getResolutionAlignmentMap(List.of(COMPLETE,QUARTERS), QUARTER)); + List contexts = DateContext.generateRelativeContexts( + event, + indexPlacement, + featureTime, + outcomeTime, + timeUnit, + ExportForm.getResolutionAlignmentMap(List.of(COMPLETE, QUARTERS), QUARTER)); assertThat(contexts).containsExactly( - new DateContext(CDateRange.of(LocalDate.of(2001, 1, 1), LocalDate.of(2001, 6, 30)), FeatureGroup.FEATURE, null, eventDate, COMPLETE), - new DateContext(CDateRange.of(LocalDate.of(2001, 7, 1), LocalDate.of(2001, 9, 30)), FeatureGroup.OUTCOME, null, eventDate, COMPLETE), - new DateContext(CDateRange.of(LocalDate.of(2001, 1, 1), LocalDate.of(2001, 3, 31)), FeatureGroup.FEATURE, -2, eventDate, QUARTERS), - new DateContext(CDateRange.of(LocalDate.of(2001, 4, 1), LocalDate.of(2001, 6, 30)), FeatureGroup.FEATURE, -1, eventDate, QUARTERS), - new DateContext(CDateRange.of(LocalDate.of(2001, 7, 1), LocalDate.of(2001, 9, 30)), FeatureGroup.OUTCOME, 1, eventDate, QUARTERS) + new DateContext( + CDateRange.of(LocalDate.of(2001, 1, 1), LocalDate.of(2001, 6, 30)), + FeatureGroup.FEATURE, + null, + eventDate, + COMPLETE), + new DateContext( + CDateRange.of(LocalDate.of(2001, 7, 1), LocalDate.of(2001, 9, 30)), + FeatureGroup.OUTCOME, + null, + eventDate, + COMPLETE), + new DateContext( + CDateRange.of(LocalDate.of(2001, 1, 1), LocalDate.of(2001, 3, 31)), + FeatureGroup.FEATURE, + -2, + eventDate, + QUARTERS), + new DateContext( + CDateRange.of(LocalDate.of(2001, 4, 1), LocalDate.of(2001, 6, 30)), + FeatureGroup.FEATURE, + -1, + eventDate, + QUARTERS), + new DateContext( + CDateRange.of(LocalDate.of(2001, 7, 1), LocalDate.of(2001, 9, 30)), + FeatureGroup.OUTCOME, + 1, + eventDate, + QUARTERS) ); } @@ -223,17 +387,51 @@ public void rangeRelQuarterAfterTest() { IndexPlacement indexPlacement = IndexPlacement.AFTER; - List - contexts = - DateContext.generateRelativeContexts(event, indexPlacement, featureTime, outcomeTime, timeUnit, ExportForm.getResolutionAlignmentMap(List.of(COMPLETE, QUARTERS), QUARTER)); + List contexts = DateContext.generateRelativeContexts( + event, + indexPlacement, + featureTime, + outcomeTime, + timeUnit, + ExportForm.getResolutionAlignmentMap(List.of(COMPLETE, QUARTERS), QUARTER)); assertThat(contexts).containsExactly( - new DateContext(CDateRange.of(LocalDate.of(2000, 10, 1), LocalDate.of(2001, 3, 31)), FeatureGroup.FEATURE, null, eventDate, COMPLETE), - new DateContext(CDateRange.of(LocalDate.of(2001, 4, 1), LocalDate.of(2001, 9, 30)), FeatureGroup.OUTCOME, null, eventDate, COMPLETE), - new DateContext(CDateRange.of(LocalDate.of(2000, 10, 1), LocalDate.of(2000, 12, 31)), FeatureGroup.FEATURE, -2, eventDate, QUARTERS), - new DateContext(CDateRange.of(LocalDate.of(2001, 1, 1), LocalDate.of(2001, 3, 31)), FeatureGroup.FEATURE, -1, eventDate, QUARTERS), - new DateContext(CDateRange.of(LocalDate.of(2001, 4, 1), LocalDate.of(2001, 6, 30)), FeatureGroup.OUTCOME, 1, eventDate, QUARTERS), - new DateContext(CDateRange.of(LocalDate.of(2001, 7, 1), LocalDate.of(2001, 9, 30)), FeatureGroup.OUTCOME, 2, eventDate, QUARTERS) + new DateContext( + CDateRange.of(LocalDate.of(2000, 10, 1), LocalDate.of(2001, 3, 31)), + FeatureGroup.FEATURE, + null, + eventDate, + COMPLETE), + new DateContext( + CDateRange.of(LocalDate.of(2001, 4, 1), LocalDate.of(2001, 9, 30)), + FeatureGroup.OUTCOME, + null, + eventDate, + COMPLETE), + new DateContext( + CDateRange.of(LocalDate.of(2000, 10, 1), LocalDate.of(2000, 12, 31)), + FeatureGroup.FEATURE, + -2, + eventDate, + QUARTERS), + new DateContext( + CDateRange.of(LocalDate.of(2001, 1, 1), LocalDate.of(2001, 3, 31)), + FeatureGroup.FEATURE, + -1, + eventDate, + QUARTERS), + new DateContext( + CDateRange.of(LocalDate.of(2001, 4, 1), LocalDate.of(2001, 6, 30)), + FeatureGroup.OUTCOME, + 1, + eventDate, + QUARTERS), + new DateContext( + CDateRange.of(LocalDate.of(2001, 7, 1), LocalDate.of(2001, 9, 30)), + FeatureGroup.OUTCOME, + 2, + eventDate, + QUARTERS) ); } @@ -247,15 +445,51 @@ public void rangeRelQuarterNeutralTest() { IndexPlacement indexPlacement = IndexPlacement.NEUTRAL; - List contexts = DateContext.generateRelativeContexts(event, indexPlacement, featureTime, outcomeTime, timeUnit, ExportForm.getResolutionAlignmentMap(List.of(COMPLETE,QUARTERS), QUARTER)); + List contexts = DateContext.generateRelativeContexts( + event, + indexPlacement, + featureTime, + outcomeTime, + timeUnit, + ExportForm.getResolutionAlignmentMap(List.of(COMPLETE, QUARTERS), QUARTER)); - assertThat(contexts).containsExactly ( - new DateContext(CDateRange.of(LocalDate.of(2000, 10, 1), LocalDate.of(2001, 3, 31)), FeatureGroup.FEATURE, null, eventDate, COMPLETE), - new DateContext(CDateRange.of(LocalDate.of(2001, 7, 1), LocalDate.of(2001, 12, 31)), FeatureGroup.OUTCOME, null, eventDate, COMPLETE), - new DateContext(CDateRange.of(LocalDate.of(2000, 10, 1), LocalDate.of(2000, 12, 31)), FeatureGroup.FEATURE, -2, eventDate, QUARTERS), - new DateContext(CDateRange.of(LocalDate.of(2001, 1, 1), LocalDate.of(2001, 3, 31)), FeatureGroup.FEATURE, -1, eventDate, QUARTERS), - new DateContext(CDateRange.of(LocalDate.of(2001, 7, 1), LocalDate.of(2001, 9, 30)), FeatureGroup.OUTCOME, 1, eventDate, QUARTERS), - new DateContext(CDateRange.of(LocalDate.of(2001, 10, 1), LocalDate.of(2001, 12, 31)), FeatureGroup.OUTCOME, 2, eventDate, QUARTERS) + assertThat(contexts).containsExactly( + new DateContext( + CDateRange.of(LocalDate.of(2000, 10, 1), LocalDate.of(2001, 3, 31)), + FeatureGroup.FEATURE, + null, + eventDate, + COMPLETE), + new DateContext( + CDateRange.of(LocalDate.of(2001, 7, 1), LocalDate.of(2001, 12, 31)), + FeatureGroup.OUTCOME, + null, + eventDate, + COMPLETE), + new DateContext( + CDateRange.of(LocalDate.of(2000, 10, 1), LocalDate.of(2000, 12, 31)), + FeatureGroup.FEATURE, + -2, + eventDate, + QUARTERS), + new DateContext( + CDateRange.of(LocalDate.of(2001, 1, 1), LocalDate.of(2001, 3, 31)), + FeatureGroup.FEATURE, + -1, + eventDate, + QUARTERS), + new DateContext( + CDateRange.of(LocalDate.of(2001, 7, 1), LocalDate.of(2001, 9, 30)), + FeatureGroup.OUTCOME, + 1, + eventDate, + QUARTERS), + new DateContext( + CDateRange.of(LocalDate.of(2001, 10, 1), LocalDate.of(2001, 12, 31)), + FeatureGroup.OUTCOME, + 2, + eventDate, + QUARTERS) ); } @@ -269,14 +503,45 @@ public void rangeRelYearsAlignQuarterNeutralTest() { IndexPlacement indexPlacement = IndexPlacement.NEUTRAL; - List contexts = DateContext.generateRelativeContexts(event, indexPlacement, featureTime, outcomeTime, timeUnit, ExportForm.getResolutionAlignmentMap(List.of(COMPLETE,YEARS), QUARTER)); + List contexts = DateContext.generateRelativeContexts( + event, + indexPlacement, + featureTime, + outcomeTime, + timeUnit, + ExportForm.getResolutionAlignmentMap(List.of(COMPLETE, YEARS), QUARTER)); - assertThat(contexts).containsExactly ( - new DateContext(CDateRange.of(LocalDate.of(2000, 10, 1), LocalDate.of(2001, 3, 31)), FeatureGroup.FEATURE, null, eventDate, COMPLETE), - new DateContext(CDateRange.of(LocalDate.of(2001, 7, 1), LocalDate.of(2002, 12, 31)), FeatureGroup.OUTCOME, null, eventDate, COMPLETE), - new DateContext(CDateRange.of(LocalDate.of(2000, 10, 1), LocalDate.of(2001, 3, 31)), FeatureGroup.FEATURE, -1, eventDate, YEARS), - new DateContext(CDateRange.of(LocalDate.of(2001, 7, 1), LocalDate.of(2002, 6, 30)), FeatureGroup.OUTCOME, 1, eventDate, YEARS), - new DateContext(CDateRange.of(LocalDate.of(2002, 7, 1), LocalDate.of(2002, 12, 31)), FeatureGroup.OUTCOME, 2, eventDate, YEARS) + assertThat(contexts).containsExactly( + new DateContext( + CDateRange.of(LocalDate.of(2000, 10, 1), LocalDate.of(2001, 3, 31)), + FeatureGroup.FEATURE, + null, + eventDate, + COMPLETE), + new DateContext( + CDateRange.of(LocalDate.of(2001, 7, 1), LocalDate.of(2002, 12, 31)), + FeatureGroup.OUTCOME, + null, + eventDate, + COMPLETE), + new DateContext( + CDateRange.of(LocalDate.of(2000, 10, 1), LocalDate.of(2001, 3, 31)), + FeatureGroup.FEATURE, + -1, + eventDate, + YEARS), + new DateContext( + CDateRange.of(LocalDate.of(2001, 7, 1), LocalDate.of(2002, 6, 30)), + FeatureGroup.OUTCOME, + 1, + eventDate, + YEARS), + new DateContext( + CDateRange.of(LocalDate.of(2002, 7, 1), LocalDate.of(2002, 12, 31)), + FeatureGroup.OUTCOME, + 2, + eventDate, + YEARS) ); } @@ -292,13 +557,39 @@ public void rangeRelDaysAlignQuarterNeutralTest() { IndexPlacement indexPlacement = IndexPlacement.NEUTRAL; - List contexts = DateContext.generateRelativeContexts(event, indexPlacement, featureTime, outcomeTime, timeUnit, ExportForm.getResolutionAlignmentMap(List.of(DAYS), QUARTER)); + List contexts = DateContext.generateRelativeContexts( + event, + indexPlacement, + featureTime, + outcomeTime, + timeUnit, + ExportForm.getResolutionAlignmentMap(List.of(DAYS), QUARTER)); - assertThat(contexts).containsExactly ( - new DateContext(CDateRange.of(LocalDate.of(2001, 5, 21), LocalDate.of(2001, 5, 21)), FeatureGroup.FEATURE, -2, eventDate, DAYS), - new DateContext(CDateRange.of(LocalDate.of(2001, 5, 22), LocalDate.of(2001, 5, 22)), FeatureGroup.FEATURE, -1, eventDate, DAYS), - new DateContext(CDateRange.of(LocalDate.of(2001, 5, 24), LocalDate.of(2001, 5, 24)), FeatureGroup.OUTCOME, 1, eventDate, DAYS), - new DateContext(CDateRange.of(LocalDate.of(2001, 5, 25), LocalDate.of(2001, 5, 25)), FeatureGroup.OUTCOME, 2, eventDate, DAYS) + assertThat(contexts).containsExactly( + new DateContext( + CDateRange.of(LocalDate.of(2001, 5, 21), LocalDate.of(2001, 5, 21)), + FeatureGroup.FEATURE, + -2, + eventDate, + DAYS), + new DateContext( + CDateRange.of(LocalDate.of(2001, 5, 22), LocalDate.of(2001, 5, 22)), + FeatureGroup.FEATURE, + -1, + eventDate, + DAYS), + new DateContext( + CDateRange.of(LocalDate.of(2001, 5, 24), LocalDate.of(2001, 5, 24)), + FeatureGroup.OUTCOME, + 1, + eventDate, + DAYS), + new DateContext( + CDateRange.of(LocalDate.of(2001, 5, 25), LocalDate.of(2001, 5, 25)), + FeatureGroup.OUTCOME, + 2, + eventDate, + DAYS) ); } @@ -315,20 +606,76 @@ public void rangeRelYearsQuarterAlignYearsNeutralTest() { IndexPlacement indexPlacement = IndexPlacement.NEUTRAL; - List contexts = DateContext.generateRelativeContexts(event, indexPlacement, featureTime, outcomeTime, timeUnit, ExportForm.getResolutionAlignmentMap(List.of(YEARS, QUARTERS), YEAR)); - - assertThat(contexts).containsExactly ( - new DateContext(CDateRange.of(LocalDate.of(2000, 7, 1), LocalDate.of(2000, 12, 31)), FeatureGroup.FEATURE, -2, eventDate, YEARS), - new DateContext(CDateRange.of(LocalDate.of(2001, 1, 1), LocalDate.of(2001, 3, 31)), FeatureGroup.FEATURE, -1, eventDate, YEARS), - new DateContext(CDateRange.of(LocalDate.of(2001, 7, 1), LocalDate.of(2001, 12, 31)), FeatureGroup.OUTCOME, 1, eventDate, YEARS), - new DateContext(CDateRange.of(LocalDate.of(2002, 1, 1), LocalDate.of(2002, 3, 31)), FeatureGroup.OUTCOME, 2, eventDate, YEARS), + List contexts = DateContext.generateRelativeContexts( + event, + indexPlacement, + featureTime, + outcomeTime, + timeUnit, + ExportForm.getResolutionAlignmentMap(List.of(YEARS, QUARTERS), YEAR)); - new DateContext(CDateRange.of(LocalDate.of(2000, 7, 1), LocalDate.of(2000, 9, 30)), FeatureGroup.FEATURE, -3, eventDate, QUARTERS), - new DateContext(CDateRange.of(LocalDate.of(2000, 10, 1), LocalDate.of(2000, 12, 31)), FeatureGroup.FEATURE, -2, eventDate, QUARTERS), - new DateContext(CDateRange.of(LocalDate.of(2001, 1, 1), LocalDate.of(2001, 3, 31)), FeatureGroup.FEATURE, -1, eventDate, QUARTERS), - new DateContext(CDateRange.of(LocalDate.of(2001, 7, 1), LocalDate.of(2001, 9, 30)), FeatureGroup.OUTCOME, 1, eventDate, QUARTERS), - new DateContext(CDateRange.of(LocalDate.of(2001, 10, 1), LocalDate.of(2001, 12, 31)), FeatureGroup.OUTCOME, 2, eventDate, QUARTERS), - new DateContext(CDateRange.of(LocalDate.of(2002, 1, 1), LocalDate.of(2002, 3, 31)), FeatureGroup.OUTCOME, 3, eventDate, QUARTERS) + assertThat(contexts).containsExactly( + new DateContext( + CDateRange.of(LocalDate.of(2000, 7, 1), LocalDate.of(2000, 12, 31)), + FeatureGroup.FEATURE, + -2, + eventDate, + YEARS), + new DateContext( + CDateRange.of(LocalDate.of(2001, 1, 1), LocalDate.of(2001, 3, 31)), + FeatureGroup.FEATURE, + -1, + eventDate, + YEARS), + new DateContext( + CDateRange.of(LocalDate.of(2001, 7, 1), LocalDate.of(2001, 12, 31)), + FeatureGroup.OUTCOME, + 1, + eventDate, + YEARS), + new DateContext( + CDateRange.of(LocalDate.of(2002, 1, 1), LocalDate.of(2002, 3, 31)), + FeatureGroup.OUTCOME, + 2, + eventDate, + YEARS), + + new DateContext( + CDateRange.of(LocalDate.of(2000, 7, 1), LocalDate.of(2000, 9, 30)), + FeatureGroup.FEATURE, + -3, + eventDate, + QUARTERS), + new DateContext( + CDateRange.of(LocalDate.of(2000, 10, 1), LocalDate.of(2000, 12, 31)), + FeatureGroup.FEATURE, + -2, + eventDate, + QUARTERS), + new DateContext( + CDateRange.of(LocalDate.of(2001, 1, 1), LocalDate.of(2001, 3, 31)), + FeatureGroup.FEATURE, + -1, + eventDate, + QUARTERS), + new DateContext( + CDateRange.of(LocalDate.of(2001, 7, 1), LocalDate.of(2001, 9, 30)), + FeatureGroup.OUTCOME, + 1, + eventDate, + QUARTERS), + new DateContext( + CDateRange.of(LocalDate.of(2001, 10, 1), LocalDate.of(2001, 12, 31)), + FeatureGroup.OUTCOME, + 2, + eventDate, + QUARTERS), + new DateContext( + CDateRange.of(LocalDate.of(2002, 1, 1), LocalDate.of(2002, 3, 31)), + FeatureGroup.OUTCOME, + 3, + eventDate, + QUARTERS) ); } } diff --git a/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapTest.java b/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapTest.java index 2203c99a716..f91474de61c 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapTest.java @@ -40,15 +40,17 @@ public void generalTest() throws IOException { storage.injectInto(mapper); JsonNode json = mapper.valueToTree(containedDataset); - + /*assertThat(json.isArray()).isTrue(); assertThat(json.get(0)).isEqualTo(Jackson.MAPPER.valueToTree(d1));*/ ContainingDataset constructed = mapper.treeToValue(json, ContainingDataset.class); assertThat(constructed.idMap.entrySet()).isEqualTo(containedDataset.idMap.entrySet()); } - - @Data @NoArgsConstructor @AllArgsConstructor + + @Data + @NoArgsConstructor + @AllArgsConstructor public static class ContainingDataset { private IdMap idMap = new IdMap<>(); } diff --git a/backend/src/test/java/com/bakdata/conquery/models/identifiable/ids/IdTests.java b/backend/src/test/java/com/bakdata/conquery/models/identifiable/ids/IdTests.java index 2b982e39812..e88b009568d 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/identifiable/ids/IdTests.java +++ b/backend/src/test/java/com/bakdata/conquery/models/identifiable/ids/IdTests.java @@ -34,48 +34,49 @@ public class IdTests { public static final NamespacedStorageProvider STORAGE = new TestNamespacedStorageProvider(); public static Stream reflectionTest() { - return CPSTypeIdResolver - .SCAN_RESULT - .getClassesImplementing(Identifiable.class.getName()).loadClasses() - .stream() - .filter(cl -> !cl.isInterface()) - .filter(cl -> !Modifier.isAbstract(cl.getModifiers())) - //filter test classes - .filter(cl -> !cl.toString().toLowerCase().contains("test")) - .map(cl -> { - - Class idClazz = null; - // Try to get the specific Id + return CPSTypeIdResolver.SCAN_RESULT.getClassesImplementing( + Identifiable.class.getName()) + .loadClasses() + .stream() + .filter(cl -> !cl.isInterface()) + .filter( + cl -> !Modifier.isAbstract(cl.getModifiers())) + //filter test classes + .filter(cl -> !cl.toString().toLowerCase().contains("test")) + .map(cl -> { + + Class idClazz = null; + // Try to get the specific Id + try { + idClazz = cl.getMethod("getId").getReturnType(); + + } catch (NoSuchMethodException e) { + return fail(cl.getName() + " does not implement the method 'getId()'"); + } + + if (Modifier.isAbstract(idClazz.getModifiers())) { try { - idClazz = cl.getMethod("getId").getReturnType(); + idClazz = cl.getMethod("createId").getReturnType(); + } catch (NoSuchMethodException e) { + return fail( + cl.getName() + " does not implement the method 'createId()' unable to retrieve specific id class"); } - catch (NoSuchMethodException e) { - return fail(cl.getName() + " does not implement the method 'getId()'"); - } - - if (Modifier.isAbstract(idClazz.getModifiers())) { - try { - idClazz = cl.getMethod("createId").getReturnType(); - - } - catch (NoSuchMethodException e) { - return fail(cl.getName() + " does not implement the method 'createId()' unable to retrieve specific id class"); - } - } - - String packageString = "com.bakdata.conquery.models.identifiable.ids.specific."; - if (!idClazz.getName().startsWith(packageString)) { - return fail("The id class " + idClazz + " is not located in the package " + packageString + ". Please clean that up."); - } - - return Arguments.of( - cl, - idClazz - ); - }); + } + + String packageString = "com.bakdata.conquery.models.identifiable.ids.specific."; + if (!idClazz.getName().startsWith(packageString)) { + return fail( + "The id class " + idClazz + " is not located in the package " + packageString + ". Please clean that up."); + } + + return Arguments.of( + cl, + idClazz + ); + }); } - + @Test public void testEquals() { ConceptTreeChildId idA = new ConceptTreeChildId( @@ -104,7 +105,7 @@ public void testEquals() { assertThat(idA).hasSameHashCodeAs(idB); assertThat(idA.toString()).isEqualTo(idB.toString()); } - + @Test public void testStringSerialization() { ConceptTreeChildId id = new ConceptTreeChildId( @@ -124,7 +125,7 @@ public void testStringSerialization() { assertThat(copy).hasSameHashCodeAs(id); assertThat(copy.toString()).isEqualTo(id.toString()); } - + @Test public void testJacksonSerialization() throws IOException { ConceptTreeChildId id = new ConceptTreeChildId( @@ -147,13 +148,18 @@ public void testJacksonSerialization() throws IOException { assertThat(copy).hasSameHashCodeAs(id); assertThat(copy.toString()).isEqualTo(id.toString()); } - + @Test public void testInterning() throws IOException { - InternalMapperFactory internalMapperFactory = new InternalMapperFactory(new ConqueryConfig(), Validators.newValidator()); + InternalMapperFactory internalMapperFactory = new InternalMapperFactory( + new ConqueryConfig(), + Validators.newValidator()); ObjectMapper objectMapper = Jackson.copyMapperAndInjectables(Jackson.MAPPER); - internalMapperFactory.customizeApiObjectMapper(objectMapper, mock(DatasetRegistry.class), new NonPersistentStoreFactory().createMetaStorage()); + internalMapperFactory.customizeApiObjectMapper( + objectMapper, + mock(DatasetRegistry.class), + new NonPersistentStoreFactory().createMetaStorage()); STORAGE.injectInto(objectMapper); // DatasetRegistry-mock doesn't properly inject itself @@ -170,7 +176,7 @@ public void testInterning() throws IOException { assertThat(id1.findConcept()).isSameAs(id2.findConcept()); assertThat(id1.findConcept().getDataset()).isSameAs(id2.findConcept().getDataset()); } - + @Test public void testJacksonBinarySerialization() throws IOException { ConceptTreeChildId id = new ConceptTreeChildId( diff --git a/backend/src/test/java/com/bakdata/conquery/models/identifiable/mapping/PseudomizationTest.java b/backend/src/test/java/com/bakdata/conquery/models/identifiable/mapping/PseudomizationTest.java index 74bfe9bd9ed..3cc27b3bc30 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/identifiable/mapping/PseudomizationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/identifiable/mapping/PseudomizationTest.java @@ -8,7 +8,7 @@ public class PseudomizationTest { @Test void pseudoIdGeneration() { - AutoIncrementingPseudomizer pseudomizer = new AutoIncrementingPseudomizer(4,2); + AutoIncrementingPseudomizer pseudomizer = new AutoIncrementingPseudomizer(4, 2); // Id changes from internal to external assertThat(pseudomizer.getPseudoId("0")).isEqualTo(EntityPrintId.from(null, null, "anon_0", null)); @@ -21,4 +21,4 @@ void pseudoIdGeneration() { } -} \ No newline at end of file +} diff --git a/backend/src/test/java/com/bakdata/conquery/models/preproc/PreprocessorTest.java b/backend/src/test/java/com/bakdata/conquery/models/preproc/PreprocessorTest.java index c0c05b7546f..1a2a3296660 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/preproc/PreprocessorTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/preproc/PreprocessorTest.java @@ -14,4 +14,4 @@ void getTaggedVersion() { assertThat(Preprocessor.getTaggedVersion(new File("file.csv"), "tag", ".csv")).hasName("file.tag.csv"); assertThat(Preprocessor.getTaggedVersion(new File("file.csv.csv"), "tag", ".csv")).hasName("file.csv.tag.csv"); } -} \ No newline at end of file +} diff --git a/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/DateRangeParserTest.java b/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/DateRangeParserTest.java index ba126fb6549..cbe63036438 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/DateRangeParserTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/DateRangeParserTest.java @@ -20,15 +20,15 @@ class DateRangeParserTest { public void onlyClosed() { final DateRangeParser parser = new DateRangeParser(new ConqueryConfig()); - List.of(CDateRange.of(10, 11), CDateRange.exactly(10)) - .forEach(parser::addLine); + List.of(CDateRange.of(10, 11), CDateRange.exactly(10)).forEach(parser::addLine); final ColumnStore actual = parser.decideType(); assertThat(actual).isInstanceOf(DirectDateRangeStore.class); - assertThat(((IntegerDateStore) ((DirectDateRangeStore) actual).getMinStore()).getStore()) - .isInstanceOfAny(ByteArrayStore.class, RebasingIntegerStore.class); + assertThat(((IntegerDateStore) ((DirectDateRangeStore) actual).getMinStore()).getStore()).isInstanceOfAny( + ByteArrayStore.class, + RebasingIntegerStore.class); } @@ -36,8 +36,7 @@ public void onlyClosed() { public void notOnlyClosed() { final DateRangeParser parser = new DateRangeParser(new ConqueryConfig()); - List.of(CDateRange.of(10, 11), CDateRange.exactly(10), CDateRange.atMost(10)) - .forEach(parser::addLine); + List.of(CDateRange.of(10, 11), CDateRange.exactly(10), CDateRange.atMost(10)).forEach(parser::addLine); assertThat(parser.decideType()).isInstanceOf(DirectDateRangeStore.class); } @@ -46,9 +45,11 @@ public void notOnlyClosed() { public void onlyQuarters() { final DateRangeParser parser = new DateRangeParser(new ConqueryConfig()); - List.of(CDateRange.of(QuarterUtils.getFirstDayOfQuarter(2011, 1), QuarterUtils.getLastDayOfQuarter(2011, 1))) - .forEach(parser::addLine); + List.of( + CDateRange.of( + QuarterUtils.getFirstDayOfQuarter(2011, 1), + QuarterUtils.getLastDayOfQuarter(2011, 1))).forEach(parser::addLine); assertThat(parser.decideType()).isInstanceOf(QuarterDateRangeStore.class); } -} \ No newline at end of file +} diff --git a/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/DecimalParserTest.java b/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/DecimalParserTest.java index 6b27a21e02a..21b623ec220 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/DecimalParserTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/DecimalParserTest.java @@ -14,11 +14,11 @@ class DecimalParserTest { @Test - public void test(){ + public void test() { final DecimalParser parser = new DecimalParser(new ConqueryConfig()); - parser.addLine(BigDecimal.valueOf(10,1000)); - parser.addLine(BigDecimal.valueOf(30,1000)); + parser.addLine(BigDecimal.valueOf(10, 1000)); + parser.addLine(BigDecimal.valueOf(30, 1000)); assertThat(parser.decideType()).isInstanceOf(ScaledDecimalStore.class); @@ -27,4 +27,4 @@ public void test(){ assertThat(parser.decideType()).isInstanceOf(DecimalArrayStore.class); } -} \ No newline at end of file +} diff --git a/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/IntegerParserTest.java b/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/IntegerParserTest.java index 417a990fab3..e75fd515bc3 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/IntegerParserTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/IntegerParserTest.java @@ -25,22 +25,28 @@ class IntegerParserTest { public static Stream arguments() { return Stream.of( - // Long is everything not match-able to other sizes. - Arguments.of(Integer.MIN_VALUE, Integer.MAX_VALUE * 2L, direct(LongArrayStore.class)), + // Long is everything not match-able to other sizes. + Arguments.of(Integer.MIN_VALUE, Integer.MAX_VALUE * 2L, direct(LongArrayStore.class)), - // IntegerStore - Arguments.of(Integer.MIN_VALUE, (long) Integer.MAX_VALUE - 1L, direct(IntArrayStore.class)), - Arguments.of(Long.MIN_VALUE, Long.MIN_VALUE + (long) Integer.MAX_VALUE - 1L, rebased(IntArrayStore.class)), + // IntegerStore + Arguments.of(Integer.MIN_VALUE, (long) Integer.MAX_VALUE - 1L, direct(IntArrayStore.class)), + Arguments.of(Long.MIN_VALUE, Long.MIN_VALUE + (long) Integer.MAX_VALUE - 1L, rebased(IntArrayStore.class)), - // ByteArrayStore - Arguments.of(Byte.MIN_VALUE, Byte.MAX_VALUE - 1L, direct(ByteArrayStore.class)), - Arguments.of(Long.MIN_VALUE, Long.MIN_VALUE + 255 - 1L, rebased(ByteArrayStore.class)), - Arguments.of(Short.MAX_VALUE, Short.MAX_VALUE + 255 - 1L, rebased(ByteArrayStore.class)), + // ByteArrayStore + Arguments.of(Byte.MIN_VALUE, Byte.MAX_VALUE - 1L, direct(ByteArrayStore.class)), + Arguments.of(Long.MIN_VALUE, Long.MIN_VALUE + 255 - 1L, rebased(ByteArrayStore.class)), + Arguments.of(Short.MAX_VALUE, Short.MAX_VALUE + 255 - 1L, rebased(ByteArrayStore.class)), - // ShortArrayStore - Arguments.of(Short.MIN_VALUE, Short.MAX_VALUE - 1L, direct(ShortArrayStore.class)), - Arguments.of(Integer.MIN_VALUE, Integer.MIN_VALUE + ((long) Short.MAX_VALUE - (long) Short.MIN_VALUE) - 1L, rebased(ShortArrayStore.class)), - Arguments.of(Short.MAX_VALUE, Short.MAX_VALUE + (Short.MAX_VALUE - Short.MIN_VALUE) - 1L, rebased(ShortArrayStore.class)) + // ShortArrayStore + Arguments.of(Short.MIN_VALUE, Short.MAX_VALUE - 1L, direct(ShortArrayStore.class)), + Arguments.of( + Integer.MIN_VALUE, + Integer.MIN_VALUE + ((long) Short.MAX_VALUE - (long) Short.MIN_VALUE) - 1L, + rebased(ShortArrayStore.class)), + Arguments.of( + Short.MAX_VALUE, + Short.MAX_VALUE + (Short.MAX_VALUE - Short.MIN_VALUE) - 1L, + rebased(ShortArrayStore.class)) ); } @@ -50,10 +56,11 @@ public static Consumer direct(Class clazz) { public static Consumer rebased(Class clazz) { return store -> { - assertThat(store) - .asInstanceOf(new InstanceOfAssertFactory<>(RebasingIntegerStore.class, new ObjectAssertFactory<>())) - .extracting(RebasingIntegerStore::getStore) - .isInstanceOf(clazz); + assertThat(store).asInstanceOf( + new InstanceOfAssertFactory<>(RebasingIntegerStore.class, new ObjectAssertFactory<>())) + .extracting( + RebasingIntegerStore::getStore) + .isInstanceOf(clazz); }; } @@ -65,7 +72,6 @@ public void test(long min, long max, Consumer test) { parser.setMinValue(min); parser.setMaxValue(max); - assertThat(parser.decideType()) - .satisfies(test); + assertThat(parser.decideType()).satisfies(test); } -} \ No newline at end of file +} diff --git a/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/RealParserTest.java b/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/RealParserTest.java index 962e312ea05..802f55cdb0f 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/RealParserTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/RealParserTest.java @@ -3,7 +3,6 @@ import static org.assertj.core.api.Assertions.assertThat; import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.config.ParserConfig; import com.bakdata.conquery.models.events.stores.primitive.DoubleArrayStore; import com.bakdata.conquery.models.events.stores.primitive.FloatArrayStore; import org.junit.jupiter.api.Test; @@ -30,4 +29,4 @@ public void ulpInRange() { assertThat(realParser.decideType()).isInstanceOf(DoubleArrayStore.class); } -} \ No newline at end of file +} diff --git a/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java b/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java index e27581b5dcb..88c7ebaac35 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java @@ -2,6 +2,7 @@ import static org.assertj.core.api.Assertions.assertThat; +import jakarta.validation.Validator; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -12,7 +13,6 @@ import java.util.function.BiFunction; import java.util.stream.Collectors; import java.util.stream.Stream; -import jakarta.validation.Validator; import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; @@ -46,111 +46,114 @@ @Slf4j public class DefaultColumnNameTest { - private static final PrintSettings SETTINGS = new PrintSettings(false, Locale.ENGLISH, new ConqueryConfig(), null, null); + private static final PrintSettings SETTINGS = new PrintSettings( + false, + Locale.ENGLISH, + new ConqueryConfig(), + null, + null); private static final Validator VALIDATOR = Validators.newValidator(); - private static final BiFunction CONCEPT_SELECT_SELECTOR = - (concept, cq) -> { - final UniversalSelect select = concept.getSelects().getFirst(); - cq.setSelects(List.of(select.getId())); - return select; - }; + private static final BiFunction CONCEPT_SELECT_SELECTOR = (concept, cq) -> { + final UniversalSelect select = concept.getSelects().getFirst(); + cq.setSelects(List.of(select.getId())); + return select; + }; - private static final BiFunction CONNECTOR_SELECT_SELECTOR = - (concept, cq) -> { - final Select select = concept.getConnectors().getFirst().getSelects().getFirst(); - cq.getTables().getFirst().setSelects(List.of((ConnectorSelectId) select.getId())); - return select; - }; + private static final BiFunction CONNECTOR_SELECT_SELECTOR = (concept, cq) -> { + final Select select = concept.getConnectors().getFirst().getSelects().getFirst(); + cq.getTables().getFirst().setSelects(List.of((ConnectorSelectId) select.getId())); + return select; + }; private static Stream provideCombinations() { return Stream.of( - // ConceptSelect, without CQLabel, one Id - Arguments.of( - TestConcept.create(1, CONCEPT_SELECT_SELECTOR, 1, null), - false, - "TestConceptLabel ID_0 TestSelectLabel" - ), - // ConceptSelect without CQLabel, multiple Ids - Arguments.of( - TestConcept.create(1, CONCEPT_SELECT_SELECTOR, 3, null), - false, - "TestConceptLabel ID_0+ID_1+ID_2 TestSelectLabel" - ), - // ConceptSelect with CQLabel, one Id - Arguments.of( - TestConcept.create(1, CONCEPT_SELECT_SELECTOR, 1, null), - true, - "TestCQLabel TestSelectLabel" - ), - // ConceptSelect with CQLabel, multiple Ids - Arguments.of( - TestConcept.create(1, CONCEPT_SELECT_SELECTOR, 3, null), - true, - "TestCQLabel TestSelectLabel" - ), - - // ConnectorSelect, without CQLabel, one Id, one Connector - Arguments.of( - TestConcept.create(1, CONNECTOR_SELECT_SELECTOR, 1, null), - false, - "TestConceptLabel ID_0 TestSelectLabel" - ), - // ConnectorSelect without CQLabel, multiple Ids, one Connector - Arguments.of( - TestConcept.create(1, CONNECTOR_SELECT_SELECTOR, 3, null), - false, - "TestConceptLabel ID_0+ID_1+ID_2 TestSelectLabel" - ), - // ConnectorSelect with CQLabel, one Id, one Connector - Arguments.of( - TestConcept.create(1, CONNECTOR_SELECT_SELECTOR, 1, null), - true, - "TestCQLabel TestSelectLabel" - ), - // ConnectorSelect with CQLabel, multiple Ids, one Connector - Arguments.of( - TestConcept.create(1, CONNECTOR_SELECT_SELECTOR, 3, null), - true, - "TestCQLabel TestSelectLabel" - ), - - // ConnectorSelect, without CQLabel, one Id, multiple Connectors - Arguments.of( - TestConcept.create(3, CONNECTOR_SELECT_SELECTOR, 1, null), - false, - "TestConceptLabel ID_0 TestConnectorLabel_0 TestSelectLabel" - ), - // ConnectorSelect without CQLabel, multiple Ids, multiple Connectors - Arguments.of( - TestConcept.create(3, CONNECTOR_SELECT_SELECTOR, 3, null), - false, - "TestConceptLabel ID_0+ID_1+ID_2 TestConnectorLabel_0 TestSelectLabel" - ), - // ConnectorSelect with CQLabel, one Id, multiple Connectors - Arguments.of( - TestConcept.create(3, CONNECTOR_SELECT_SELECTOR, 1, null), - true, - "TestCQLabel TestConnectorLabel_0 TestSelectLabel" - ), - // ConnectorSelect with CQLabel, multiple Ids, multiple Connectors - Arguments.of( - TestConcept.create(3, CONNECTOR_SELECT_SELECTOR, 3, null), - true, - "TestCQLabel TestConnectorLabel_0 TestSelectLabel" - ), - // ConnectorSelect without CQLabel, only root Id, one Connector -> Connector label should be suppressed - Arguments.of( - TestConcept.create(1, CONNECTOR_SELECT_SELECTOR, 0, "Test-Label"), - false, - "Test-Label TestSelectLabel" - ), - Arguments.of( - TestConcept.create(1, CONCEPT_SELECT_SELECTOR, 0, "TestSelectLabel"), - false, - "TestSelectLabel" - ) + // ConceptSelect, without CQLabel, one Id + Arguments.of( + TestConcept.create(1, CONCEPT_SELECT_SELECTOR, 1, null), + false, + "TestConceptLabel ID_0 TestSelectLabel" + ), + // ConceptSelect without CQLabel, multiple Ids + Arguments.of( + TestConcept.create(1, CONCEPT_SELECT_SELECTOR, 3, null), + false, + "TestConceptLabel ID_0+ID_1+ID_2 TestSelectLabel" + ), + // ConceptSelect with CQLabel, one Id + Arguments.of( + TestConcept.create(1, CONCEPT_SELECT_SELECTOR, 1, null), + true, + "TestCQLabel TestSelectLabel" + ), + // ConceptSelect with CQLabel, multiple Ids + Arguments.of( + TestConcept.create(1, CONCEPT_SELECT_SELECTOR, 3, null), + true, + "TestCQLabel TestSelectLabel" + ), + + // ConnectorSelect, without CQLabel, one Id, one Connector + Arguments.of( + TestConcept.create(1, CONNECTOR_SELECT_SELECTOR, 1, null), + false, + "TestConceptLabel ID_0 TestSelectLabel" + ), + // ConnectorSelect without CQLabel, multiple Ids, one Connector + Arguments.of( + TestConcept.create(1, CONNECTOR_SELECT_SELECTOR, 3, null), + false, + "TestConceptLabel ID_0+ID_1+ID_2 TestSelectLabel" + ), + // ConnectorSelect with CQLabel, one Id, one Connector + Arguments.of( + TestConcept.create(1, CONNECTOR_SELECT_SELECTOR, 1, null), + true, + "TestCQLabel TestSelectLabel" + ), + // ConnectorSelect with CQLabel, multiple Ids, one Connector + Arguments.of( + TestConcept.create(1, CONNECTOR_SELECT_SELECTOR, 3, null), + true, + "TestCQLabel TestSelectLabel" + ), + + // ConnectorSelect, without CQLabel, one Id, multiple Connectors + Arguments.of( + TestConcept.create(3, CONNECTOR_SELECT_SELECTOR, 1, null), + false, + "TestConceptLabel ID_0 TestConnectorLabel_0 TestSelectLabel" + ), + // ConnectorSelect without CQLabel, multiple Ids, multiple Connectors + Arguments.of( + TestConcept.create(3, CONNECTOR_SELECT_SELECTOR, 3, null), + false, + "TestConceptLabel ID_0+ID_1+ID_2 TestConnectorLabel_0 TestSelectLabel" + ), + // ConnectorSelect with CQLabel, one Id, multiple Connectors + Arguments.of( + TestConcept.create(3, CONNECTOR_SELECT_SELECTOR, 1, null), + true, + "TestCQLabel TestConnectorLabel_0 TestSelectLabel" + ), + // ConnectorSelect with CQLabel, multiple Ids, multiple Connectors + Arguments.of( + TestConcept.create(3, CONNECTOR_SELECT_SELECTOR, 3, null), + true, + "TestCQLabel TestConnectorLabel_0 TestSelectLabel" + ), + // ConnectorSelect without CQLabel, only root Id, one Connector -> Connector label should be suppressed + Arguments.of( + TestConcept.create(1, CONNECTOR_SELECT_SELECTOR, 0, "Test-Label"), + false, + "Test-Label TestSelectLabel" + ), + Arguments.of( + TestConcept.create(1, CONCEPT_SELECT_SELECTOR, 0, "TestSelectLabel"), + false, + "TestSelectLabel" + ) ); } @@ -161,7 +164,10 @@ void checkCombinations(TestConcept concept, boolean hasCQConceptLabel, String ex final CQConcept cqConcept = concept.createCQConcept(hasCQConceptLabel); final UniqueNamer uniqNamer = new UniqueNamer(SETTINGS); - SelectResultInfo info = new SelectResultInfo(concept.extractSelect(cqConcept), cqConcept, Collections.emptySet()); + SelectResultInfo info = new SelectResultInfo( + concept.extractSelect(cqConcept), + cqConcept, + Collections.emptySet()); assertThat(uniqNamer.getUniqueName(info, SETTINGS)).isEqualTo(expectedColumnName); } @@ -175,24 +181,27 @@ private static CQConcept create(boolean withLabel, TestConcept concept) { } - List> elements = concept.getChildren().stream() - .sorted(Comparator.comparing(ctc -> ctc.getId().toString())) - .collect(Collectors.toList()); + List> elements = concept.getChildren() + .stream() + .sorted( + Comparator.comparing(ctc -> ctc.getId().toString())) + .collect(Collectors.toList()); if (elements.isEmpty()) { elements = List.of(concept); } - final List> list = elements.stream().>map(ConceptElement::getId).toList(); + final List> list = elements.stream() + .>map( + ConceptElement::getId) + .toList(); cqConcept.setElements(list); - List tables = concept.getConnectors().stream() - .map(con -> { - CQTable table = new CQTable(); - table.setConnector(con.getId()); - table.setConcept(cqConcept); - return table; - }) - .collect(Collectors.toList()); + List tables = concept.getConnectors().stream().map(con -> { + CQTable table = new CQTable(); + table.setConnector(con.getId()); + table.setConcept(cqConcept); + return table; + }).collect(Collectors.toList()); cqConcept.setTables(tables); concept.extractSelect(cqConcept); @@ -233,7 +242,11 @@ private TestConcept(BiFunction selectExtractor) } @SneakyThrows - public static TestConcept create(int countConnectors, BiFunction selectExtractor, int countIds, String overwriteLabel) { + public static TestConcept create( + int countConnectors, + BiFunction selectExtractor, + int countIds, + String overwriteLabel) { TestConcept concept = new TestConcept(selectExtractor); if (overwriteLabel != null) { concept.setLabel(overwriteLabel); diff --git a/backend/src/test/java/com/bakdata/conquery/models/query/statistics/HistogramTest.java b/backend/src/test/java/com/bakdata/conquery/models/query/statistics/HistogramTest.java index 1d878c525a7..f1d538bd73e 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/query/statistics/HistogramTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/query/statistics/HistogramTest.java @@ -19,8 +19,14 @@ class HistogramTest { @Test void recon() { - final Histogram histogram = Histogram.zeroAligned(0.0021445979668993976d, 0.9999775467040908d, 0.0021445979668993976, 0.9999775467040908d, 10, false, - false + final Histogram histogram = Histogram.zeroAligned( + 0.0021445979668993976d, + 0.9999775467040908d, + 0.0021445979668993976, + 0.9999775467040908d, + 10, + false, + false ); histogram.add(0.9999775467040908d); @@ -129,4 +135,4 @@ public void zeroWidth() { } -} \ No newline at end of file +} diff --git a/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java b/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java index 641b3d6f4a1..a6751c5f14c 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/types/ResultTypeTest.java @@ -46,69 +46,85 @@ public class ResultTypeTest { @SuppressWarnings("unused") public static List testData() { - return List.of(Arguments.of(PRETTY, ResultType.Primitive.BOOLEAN, true, "Yes"), - Arguments.of(PRETTY, ResultType.Primitive.BOOLEAN, false, "No"), - Arguments.of(PRETTY, ResultType.Primitive.STRING, "test", "test"), - Arguments.of(PRETTY, ResultType.Primitive.DATE, CDate.ofLocalDate(LocalDate.of(2013, 7, 12)), "2013-07-12"), - Arguments.of(PRETTY_DE, ResultType.Primitive.DATE, CDate.ofLocalDate(LocalDate.of(2013, 7, 12)), "12.07.2013"), - Arguments.of(PRETTY, - ResultType.Primitive.DATE_RANGE, - List.of(CDate.ofLocalDate(LocalDate.of(2013, 7, 12)), CDate.ofLocalDate(LocalDate.of(2013, 7, 12))), - "2013-07-12" - ), - Arguments.of(PRETTY_DE, - ResultType.Primitive.DATE_RANGE, - List.of(CDate.ofLocalDate(LocalDate.of(2013, 7, 12)), CDate.ofLocalDate(LocalDate.of(2013, 7, 12))), - "12.07.2013" - ), - Arguments.of(PRETTY, - ResultType.Primitive.DATE_RANGE, - List.of(CDate.ofLocalDate(LocalDate.of(2013, 7, 12)), CDate.ofLocalDate(LocalDate.of(2014, 7, 12))), - "2013-07-12/2014-07-12" - ), - Arguments.of(PRETTY_DE, - ResultType.Primitive.DATE_RANGE, - List.of(CDate.ofLocalDate(LocalDate.of(2013, 7, 12)), CDate.ofLocalDate(LocalDate.of(2014, 7, 12))), - "12.07.2013 - 12.07.2014" - ), - Arguments.of(PRETTY, ResultType.Primitive.INTEGER, 51839274L, "51,839,274"), - Arguments.of(PRETTY_DE, ResultType.Primitive.INTEGER, 51839274L, "51.839.274"), - Arguments.of(PRETTY, ResultType.Primitive.MONEY, new BigDecimal("518392.74"), "€518,392.74"), - Arguments.of(PRETTY_DE, ResultType.Primitive.MONEY, new BigDecimal("518392.74"), "518.392,74\u00A0€"), - Arguments.of(PRETTY, ResultType.Primitive.NUMERIC, 0.2, "0.2"), - Arguments.of(PRETTY_DE, ResultType.Primitive.NUMERIC, 0.2, "0,2"), - Arguments.of(PRETTY, - ResultType.Primitive.NUMERIC, - new BigDecimal("716283712389817246892743124.12312"), - "716,283,712,389,817,246,892,743,124.12312" - ), - Arguments.of(PRETTY_DE, - ResultType.Primitive.NUMERIC, - new BigDecimal("716283712389817246892743124.12312"), - "716.283.712.389.817.246.892.743.124,12312" - ), - Arguments.of(PRETTY, ResultType.Primitive.STRING, "test", "test"), - - Arguments.of(PLAIN, ResultType.Primitive.BOOLEAN, true, "1"), - Arguments.of(PLAIN, ResultType.Primitive.BOOLEAN, false, "0"), - Arguments.of(PLAIN, ResultType.Primitive.STRING, "test", "test"), - Arguments.of(PLAIN, ResultType.Primitive.DATE, LocalDate.of(2013, 7, 12).toEpochDay(), "2013-07-12"), - Arguments.of(PLAIN, ResultType.Primitive.INTEGER, 51839274L, "51839274"), - Arguments.of(PLAIN, ResultType.Primitive.MONEY, new BigDecimal(51839274L), "51839274"), - Arguments.of(PLAIN, ResultType.Primitive.NUMERIC, 0.2, "0.2"), - Arguments.of(PLAIN, - ResultType.Primitive.NUMERIC, - new BigDecimal("716283712389817246892743124.12312"), - "716283712389817246892743124.12312" - ), - Arguments.of(PLAIN, ResultType.Primitive.STRING, "test", "test"), - Arguments.of(PLAIN, ResultType.Primitive.STRING, Resolution.COMPLETE.name(), "COMPLETE") + return List.of( + Arguments.of(PRETTY, ResultType.Primitive.BOOLEAN, true, "Yes"), + Arguments.of(PRETTY, ResultType.Primitive.BOOLEAN, false, "No"), + Arguments.of(PRETTY, ResultType.Primitive.STRING, "test", "test"), + Arguments.of(PRETTY, ResultType.Primitive.DATE, CDate.ofLocalDate(LocalDate.of(2013, 7, 12)), "2013-07-12"), + Arguments.of( + PRETTY_DE, + ResultType.Primitive.DATE, + CDate.ofLocalDate(LocalDate.of(2013, 7, 12)), + "12.07.2013"), + Arguments.of( + PRETTY, + ResultType.Primitive.DATE_RANGE, + List.of(CDate.ofLocalDate(LocalDate.of(2013, 7, 12)), CDate.ofLocalDate(LocalDate.of(2013, 7, 12))), + "2013-07-12" + ), + Arguments.of( + PRETTY_DE, + ResultType.Primitive.DATE_RANGE, + List.of(CDate.ofLocalDate(LocalDate.of(2013, 7, 12)), CDate.ofLocalDate(LocalDate.of(2013, 7, 12))), + "12.07.2013" + ), + Arguments.of( + PRETTY, + ResultType.Primitive.DATE_RANGE, + List.of(CDate.ofLocalDate(LocalDate.of(2013, 7, 12)), CDate.ofLocalDate(LocalDate.of(2014, 7, 12))), + "2013-07-12/2014-07-12" + ), + Arguments.of( + PRETTY_DE, + ResultType.Primitive.DATE_RANGE, + List.of(CDate.ofLocalDate(LocalDate.of(2013, 7, 12)), CDate.ofLocalDate(LocalDate.of(2014, 7, 12))), + "12.07.2013 - 12.07.2014" + ), + Arguments.of(PRETTY, ResultType.Primitive.INTEGER, 51839274L, "51,839,274"), + Arguments.of(PRETTY_DE, ResultType.Primitive.INTEGER, 51839274L, "51.839.274"), + Arguments.of(PRETTY, ResultType.Primitive.MONEY, new BigDecimal("518392.74"), "€518,392.74"), + Arguments.of(PRETTY_DE, ResultType.Primitive.MONEY, new BigDecimal("518392.74"), "518.392,74\u00A0€"), + Arguments.of(PRETTY, ResultType.Primitive.NUMERIC, 0.2, "0.2"), + Arguments.of(PRETTY_DE, ResultType.Primitive.NUMERIC, 0.2, "0,2"), + Arguments.of( + PRETTY, + ResultType.Primitive.NUMERIC, + new BigDecimal("716283712389817246892743124.12312"), + "716,283,712,389,817,246,892,743,124.12312" + ), + Arguments.of( + PRETTY_DE, + ResultType.Primitive.NUMERIC, + new BigDecimal("716283712389817246892743124.12312"), + "716.283.712.389.817.246.892.743.124,12312" + ), + Arguments.of(PRETTY, ResultType.Primitive.STRING, "test", "test"), + + Arguments.of(PLAIN, ResultType.Primitive.BOOLEAN, true, "1"), + Arguments.of(PLAIN, ResultType.Primitive.BOOLEAN, false, "0"), + Arguments.of(PLAIN, ResultType.Primitive.STRING, "test", "test"), + Arguments.of(PLAIN, ResultType.Primitive.DATE, LocalDate.of(2013, 7, 12).toEpochDay(), "2013-07-12"), + Arguments.of(PLAIN, ResultType.Primitive.INTEGER, 51839274L, "51839274"), + Arguments.of(PLAIN, ResultType.Primitive.MONEY, new BigDecimal(51839274L), "51839274"), + Arguments.of(PLAIN, ResultType.Primitive.NUMERIC, 0.2, "0.2"), + Arguments.of( + PLAIN, + ResultType.Primitive.NUMERIC, + new BigDecimal("716283712389817246892743124.12312"), + "716283712389817246892743124.12312" + ), + Arguments.of(PLAIN, ResultType.Primitive.STRING, "test", "test"), + Arguments.of(PLAIN, ResultType.Primitive.STRING, Resolution.COMPLETE.name(), "COMPLETE") ); } @ParameterizedTest(name = "{0} {1}: {2} -> {3}") @MethodSource("testData") - public void testPrinting(PrintSettings printSettings, ResultType type, Object value, String expected) throws IOException { + public void testPrinting( + PrintSettings printSettings, + ResultType type, + Object value, + String expected) throws IOException { ResultInfo info = info(type); @@ -128,7 +144,11 @@ public static ResultInfo info(ResultType type) { @ParameterizedTest(name = "{1}: {2}") @MethodSource("testData") - public void testBinaryPrinting(PrintSettings printSettings, ResultType type, Object value, String expected) throws IOException { + public void testBinaryPrinting( + PrintSettings printSettings, + ResultType type, + Object value, + String expected) throws IOException { ResultInfo info = info(type); final Printer printer = info.createPrinter(PRINTERS, printSettings); diff --git a/backend/src/test/java/com/bakdata/conquery/resources/api/ConceptsProcessorTest.java b/backend/src/test/java/com/bakdata/conquery/resources/api/ConceptsProcessorTest.java index 86a68b1315f..b6e6e504fc5 100644 --- a/backend/src/test/java/com/bakdata/conquery/resources/api/ConceptsProcessorTest.java +++ b/backend/src/test/java/com/bakdata/conquery/resources/api/ConceptsProcessorTest.java @@ -26,7 +26,8 @@ public void testCursor() { assertThat(cursor.get(0, 10)).isEqualTo(IntStream.rangeClosed(0, 10).boxed().collect(Collectors.toList())); - assertThat(cursor.get(0, Integer.MAX_VALUE)).isEqualTo(IntStream.rangeClosed(0, 10).boxed().collect(Collectors.toList())); + assertThat(cursor.get(0, Integer.MAX_VALUE)).isEqualTo( + IntStream.rangeClosed(0, 10).boxed().collect(Collectors.toList())); } @@ -41,4 +42,4 @@ public void mapInsertion() { assertThat(map).hasSize(1); } -} \ No newline at end of file +} diff --git a/backend/src/test/java/com/bakdata/conquery/resources/api/FilterResourceTest.java b/backend/src/test/java/com/bakdata/conquery/resources/api/FilterResourceTest.java index d38d39bc34c..c2344546671 100644 --- a/backend/src/test/java/com/bakdata/conquery/resources/api/FilterResourceTest.java +++ b/backend/src/test/java/com/bakdata/conquery/resources/api/FilterResourceTest.java @@ -4,11 +4,11 @@ import static com.bakdata.conquery.resources.api.FilterResource.MAX_AUTOCOMPLETE_TEXT_LENGTH; import static org.assertj.core.api.Assertions.assertThat; -import java.util.Optional; -import java.util.OptionalInt; import jakarta.validation.ConstraintViolation; import jakarta.validation.Validation; import jakarta.validation.Validator; +import java.util.Optional; +import java.util.OptionalInt; import org.junit.jupiter.api.Test; @@ -19,9 +19,9 @@ class FilterResourceTest { @Test void acceptsAutocompleteRequestAtLimits() { final FilterResource.AutocompleteRequest request = new FilterResource.AutocompleteRequest( - Optional.of("a".repeat(MAX_AUTOCOMPLETE_TEXT_LENGTH)), - OptionalInt.empty(), - OptionalInt.of(MAX_AUTOCOMPLETE_PAGE_SIZE) + Optional.of("a".repeat(MAX_AUTOCOMPLETE_TEXT_LENGTH)), + OptionalInt.empty(), + OptionalInt.of(MAX_AUTOCOMPLETE_PAGE_SIZE) ); assertThat(VALIDATOR.validate(request)).isEmpty(); @@ -30,37 +30,37 @@ void acceptsAutocompleteRequestAtLimits() { @Test void rejectsAutocompleteTextExceedingLimit() { final FilterResource.AutocompleteRequest request = new FilterResource.AutocompleteRequest( - Optional.of("a".repeat(MAX_AUTOCOMPLETE_TEXT_LENGTH + 1)), - OptionalInt.empty(), - OptionalInt.empty() + Optional.of("a".repeat(MAX_AUTOCOMPLETE_TEXT_LENGTH + 1)), + OptionalInt.empty(), + OptionalInt.empty() ); - assertThat(VALIDATOR.validate(request)) - .singleElement() - .extracting(ConstraintViolation::getPropertyPath) - .hasToString("text"); + assertThat(VALIDATOR.validate(request)).singleElement() + .extracting( + ConstraintViolation::getPropertyPath) + .hasToString("text"); } @Test void rejectsAutocompletePageSizeExceedingLimit() { final FilterResource.AutocompleteRequest request = new FilterResource.AutocompleteRequest( - Optional.empty(), - OptionalInt.empty(), - OptionalInt.of(MAX_AUTOCOMPLETE_PAGE_SIZE + 1) + Optional.empty(), + OptionalInt.empty(), + OptionalInt.of(MAX_AUTOCOMPLETE_PAGE_SIZE + 1) ); - assertThat(VALIDATOR.validate(request)) - .singleElement() - .extracting(ConstraintViolation::getPropertyPath) - .hasToString("pageSize"); + assertThat(VALIDATOR.validate(request)).singleElement() + .extracting( + ConstraintViolation::getPropertyPath) + .hasToString("pageSize"); } @Test void acceptsEmptyAutocompleteOptions() { final FilterResource.AutocompleteRequest request = new FilterResource.AutocompleteRequest( - Optional.empty(), - OptionalInt.empty(), - OptionalInt.empty() + Optional.empty(), + OptionalInt.empty(), + OptionalInt.empty() ); assertThat(VALIDATOR.validate(request)).isEmpty(); diff --git a/backend/src/test/java/com/bakdata/conquery/service/IndexServiceTest.java b/backend/src/test/java/com/bakdata/conquery/service/IndexServiceTest.java index e695b5fbb59..76a0206511b 100644 --- a/backend/src/test/java/com/bakdata/conquery/service/IndexServiceTest.java +++ b/backend/src/test/java/com/bakdata/conquery/service/IndexServiceTest.java @@ -40,12 +40,17 @@ @Slf4j public class IndexServiceTest { @RegisterExtension - private static final MockServerExtension REF_SERVER = new MockServerExtension(ClientAndServer.startClientAndServer(), IndexServiceTest::initRefServer); + private static final MockServerExtension REF_SERVER = new MockServerExtension( + ClientAndServer.startClientAndServer(), + IndexServiceTest::initRefServer); public static final String MAPPING_PATH = "/tests/aggregator/MAPPED/mapping.csv"; - private static final NamespaceStorage NAMESPACE_STORAGE = new NamespaceStorage(new NonPersistentStoreFactory(), IndexServiceTest.class.getName()); - public static final NamespacedStorageProvider STORAGE_PROVIDER = new TestNamespacedStorageProvider(NAMESPACE_STORAGE); + private static final NamespaceStorage NAMESPACE_STORAGE = new NamespaceStorage( + new NonPersistentStoreFactory(), + IndexServiceTest.class.getName()); + public static final NamespacedStorageProvider STORAGE_PROVIDER = new TestNamespacedStorageProvider( + NAMESPACE_STORAGE); private static final Dataset DATASET = new Dataset("dataset"); private static final ConqueryConfig CONFIG = new ConqueryConfig(); @@ -56,7 +61,11 @@ public static void initRefServer(ClientAndServer mockServer) { try (InputStream inputStream = LoadingUtil.openResource(MAPPING_PATH)) { mockServer.when(request().withPath("/mapping.csv")) - .respond(HttpResponse.response().withContentType(new MediaType("text", "csv")).withBody(inputStream.readAllBytes())); + .respond( + HttpResponse.response() + .withContentType(new MediaType("text", "csv")) + .withBody( + inputStream.readAllBytes())); } } @@ -81,32 +90,36 @@ void testLoading() throws NoSuchFieldException, IllegalAccessException, URISynta try (InputStream inputStream = LoadingUtil.openResource(MAPPING_PATH)) { REF_SERVER.when(request().withPath("/mapping.csv")) - .respond(HttpResponse.response().withContentType(new MediaType("text", "csv")).withBody(inputStream.readAllBytes())); + .respond( + HttpResponse.response() + .withContentType(new MediaType("text", "csv")) + .withBody( + inputStream.readAllBytes())); } final MapInternToExternMapper mapper = new MapInternToExternMapper( - "test1", - new URI("classpath:"+MAPPING_PATH), - "internal", - "{{external}}", - false + "test1", + new URI("classpath:" + MAPPING_PATH), + "internal", + "{{external}}", + false ); final MapInternToExternMapper mapperUrlAbsolute = new MapInternToExternMapper( - "testUrlAbsolute", - new URI(String.format("http://localhost:%d/mapping.csv", REF_SERVER.getPort())), - "internal", - "{{external}}", - false + "testUrlAbsolute", + new URI(String.format("http://localhost:%d/mapping.csv", REF_SERVER.getPort())), + "internal", + "{{external}}", + false ); final MapInternToExternMapper mapperUrlRelative = new MapInternToExternMapper( - "testUrlRelative", - new URI("./mapping.csv"), - "internal", - "{{external}}", - false + "testUrlRelative", + new URI("./mapping.csv"), + "internal", + "{{external}}", + false ); @@ -132,11 +145,13 @@ void testLoading() throws NoSuchFieldException, IllegalAccessException, URISynta } - private static void injectComponents(MapInternToExternMapper mapInternToExternMapper, IndexService indexService) - throws NoSuchFieldException, IllegalAccessException { + private static void injectComponents( + MapInternToExternMapper mapInternToExternMapper, + IndexService indexService) throws NoSuchFieldException, IllegalAccessException { - final Field indexServiceField = MapInternToExternMapper.class.getDeclaredField(MapInternToExternMapper.Fields.mapIndex); + final Field indexServiceField = MapInternToExternMapper.class.getDeclaredField( + MapInternToExternMapper.Fields.mapIndex); indexServiceField.setAccessible(true); indexServiceField.set(mapInternToExternMapper, indexService); @@ -148,15 +163,14 @@ private static void injectComponents(MapInternToExternMapper mapInternToExternMa @Test @Order(2) - void testEvictOnMapper() - throws NoSuchFieldException, IllegalAccessException, URISyntaxException, ExecutionException, InterruptedException { + void testEvictOnMapper() throws NoSuchFieldException, IllegalAccessException, URISyntaxException, ExecutionException, InterruptedException { log.info("Test evicting of mapping on mapper"); final MapInternToExternMapper mapInternToExternMapper = new MapInternToExternMapper( - "test1", - new URI("classpath:"+MAPPING_PATH), - "internal", - "{{external}}", - false + "test1", + new URI("classpath:" + MAPPING_PATH), + "internal", + "{{external}}", + false ); injectComponents(mapInternToExternMapper, indexService); @@ -179,7 +193,6 @@ void testEvictOnMapper() final Index mappingAfterEvict = mapInternToExternMapper.getInt2ext().get(); // Check that the mapping reinitialized - assertThat(mappingBeforeEvict).as("Mapping before and after eviction") - .isNotSameAs(mappingAfterEvict); + assertThat(mappingBeforeEvict).as("Mapping before and after eviction").isNotSameAs(mappingAfterEvict); } } diff --git a/backend/src/test/java/com/bakdata/conquery/service/InternalFilterSearchTest.java b/backend/src/test/java/com/bakdata/conquery/service/InternalFilterSearchTest.java index 4b6bf9d6ffc..e8bb5363b89 100644 --- a/backend/src/test/java/com/bakdata/conquery/service/InternalFilterSearchTest.java +++ b/backend/src/test/java/com/bakdata/conquery/service/InternalFilterSearchTest.java @@ -1,5 +1,10 @@ package com.bakdata.conquery.service; +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import java.util.Map; + import com.bakdata.conquery.io.storage.NamespacedStorage; import com.bakdata.conquery.models.config.search.InternalSearchConfig; import com.bakdata.conquery.models.datasets.Column; @@ -17,17 +22,13 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; -import java.util.List; -import java.util.Map; - -import static org.assertj.core.api.Assertions.assertThat; - public class InternalFilterSearchTest { @RegisterExtension private static final NamespaceStorageExtension NAMESPACE_STORAGE_EXTENSION = new NamespaceStorageExtension(); private static final NamespacedStorage NAMESPACED_STORAGE = NAMESPACE_STORAGE_EXTENSION.getStorage(); - public static final NamespacedStorageProvider STORAGE_PROVIDER = new TestNamespacedStorageProvider(NAMESPACED_STORAGE); + public static final NamespacedStorageProvider STORAGE_PROVIDER = new TestNamespacedStorageProvider( + NAMESPACED_STORAGE); @Test public void totals() throws Exception { @@ -64,22 +65,27 @@ public void totals() throws Exception { // Map Searchable - filter.setLabels(ImmutableBiMap.of( - "mm", "MM", - "nn", "NN" - )); + filter.setLabels( + ImmutableBiMap.of( + "mm", + "MM", + "nn", + "NN" + )); // Register filter.getSearchReferences().forEach(searchable -> { search.addSearches(Map.of(searchable, searchConfig.createSearch(searchable))); }); - search.registerValues(column, List.of( + search.registerValues( + column, + List.of( "a", "bb", "cc", "mm" - )); + )); search.finalizeSearch(column); assertThat(search.getTotal(filter)).isEqualTo(5); diff --git a/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java b/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java index d324841bd64..3b4371157f3 100644 --- a/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java +++ b/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java @@ -29,114 +29,129 @@ class PermissionCleanupTaskTest { - private static final MetaStorage STORAGE = new NonPersistentStoreFactory().createMetaStorage(); + private static final MetaStorage STORAGE = new NonPersistentStoreFactory().createMetaStorage(); - @AfterEach - public void teardownAfterEach() { - STORAGE.clear(); - } + @AfterEach + public void teardownAfterEach() { + STORAGE.clear(); + } - @Test - void doNotDeletePermissionValidReference() { - assertThat(STORAGE.getAllExecutions()).isEmpty(); + @Test + void doNotDeletePermissionValidReference() { + assertThat(STORAGE.getAllExecutions()).isEmpty(); - final ManagedQuery managedQuery = createManagedQuery(); - // Saving the Execution - User user = new User("test", "test", STORAGE); - STORAGE.updateUser(user); - user.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managedQuery.getId())); + final ManagedQuery managedQuery = createManagedQuery(); + // Saving the Execution + User user = new User("test", "test", STORAGE); + STORAGE.updateUser(user); + user.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managedQuery.getId())); - deleteQueryPermissionsWithMissingRef(STORAGE, STORAGE.getAllUsers()); + deleteQueryPermissionsWithMissingRef(STORAGE, STORAGE.getAllUsers()); - assertThat(user.getPermissions()).containsOnly(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managedQuery.getId())); + assertThat(user.getPermissions()).containsOnly( + ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managedQuery.getId())); - } + } - private ManagedQuery createManagedQuery() { - final CQAnd root = new CQAnd(); - root.setChildren(new ArrayList<>()); + private ManagedQuery createManagedQuery() { + final CQAnd root = new CQAnd(); + root.setChildren(new ArrayList<>()); - ConceptQuery query = new ConceptQuery(root); + ConceptQuery query = new ConceptQuery(root); - final ManagedQuery managedQuery = new ManagedQuery(query, new UserId("test_user"), new DatasetId("test"), STORAGE, null, null); + final ManagedQuery managedQuery = new ManagedQuery( + query, + new UserId("test_user"), + new DatasetId("test"), + STORAGE, + null, + null); - managedQuery.setCreationTime(LocalDateTime.now().minusDays(1)); + managedQuery.setCreationTime(LocalDateTime.now().minusDays(1)); - STORAGE.addExecution(managedQuery); + STORAGE.addExecution(managedQuery); - return managedQuery; - } + return managedQuery; + } - @Test - void doDeletePermissionInvalidReference() { - assertThat(STORAGE.getAllExecutions()).isEmpty(); + @Test + void doDeletePermissionInvalidReference() { + assertThat(STORAGE.getAllExecutions()).isEmpty(); - final ManagedQuery managedQuery = createManagedQuery(); - // Removing the execution - STORAGE.removeExecution(managedQuery.getId()); - User user = new User("test", "test", STORAGE); - STORAGE.updateUser(user); - user.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managedQuery.getId())); + final ManagedQuery managedQuery = createManagedQuery(); + // Removing the execution + STORAGE.removeExecution(managedQuery.getId()); + User user = new User("test", "test", STORAGE); + STORAGE.updateUser(user); + user.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managedQuery.getId())); - deleteQueryPermissionsWithMissingRef(STORAGE, STORAGE.getAllUsers()); + deleteQueryPermissionsWithMissingRef(STORAGE, STORAGE.getAllUsers()); - assertThat(user.getPermissions()).isEmpty(); + assertThat(user.getPermissions()).isEmpty(); - } + } - @Test - void doDeletePartialPermissionWithInvalidReference() { - assertThat(STORAGE.getAllExecutions()).isEmpty(); + @Test + void doDeletePartialPermissionWithInvalidReference() { + assertThat(STORAGE.getAllExecutions()).isEmpty(); - final ManagedQuery managedQuery1 = createManagedQuery(); - final ManagedQuery managedQuery2 = createManagedQuery(); - // Removing the second execution - STORAGE.removeExecution(managedQuery2.getId()); - User user = new User("test", "test", STORAGE); - STORAGE.updateUser(user); - user.addPermission( - // Build a permission with multiple instances - new WildcardPermission(List.of( - Set.of(ExecutionPermission.DOMAIN), - Set.of(Ability.READ.toString().toLowerCase()), - Set.of(managedQuery1.getId().toString(), managedQuery2.getId().toString())), Instant.now())); + final ManagedQuery managedQuery1 = createManagedQuery(); + final ManagedQuery managedQuery2 = createManagedQuery(); + // Removing the second execution + STORAGE.removeExecution(managedQuery2.getId()); + User user = new User("test", "test", STORAGE); + STORAGE.updateUser(user); + user.addPermission( + // Build a permission with multiple instances + new WildcardPermission( + List.of( + Set.of(ExecutionPermission.DOMAIN), + Set.of(Ability.READ.toString().toLowerCase()), + Set.of(managedQuery1.getId().toString(), managedQuery2.getId().toString())), + Instant.now())); - deleteQueryPermissionsWithMissingRef(STORAGE, STORAGE.getAllUsers()); + deleteQueryPermissionsWithMissingRef(STORAGE, STORAGE.getAllUsers()); - assertThat(user.getPermissions()).containsOnly(ExecutionPermission.onInstance(Ability.READ, managedQuery1.getId())); + assertThat(user.getPermissions()).containsOnly( + ExecutionPermission.onInstance(Ability.READ, managedQuery1.getId())); - } + } - @Test - void doDeletePermissionsOfOwnedReference() { - assertThat(STORAGE.getAllExecutions()).isEmpty(); + @Test + void doDeletePermissionsOfOwnedReference() { + assertThat(STORAGE.getAllExecutions()).isEmpty(); - // Created owned execution - final ManagedQuery managedQueryOwned = createManagedQuery(); - // Setup user + // Created owned execution + final ManagedQuery managedQueryOwned = createManagedQuery(); + // Setup user User user = new User("test", "test", STORAGE); User user2 = new User("test2", "test2", STORAGE); - STORAGE.updateUser(user); - user.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managedQueryOwned.getId())); + STORAGE.updateUser(user); + user.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managedQueryOwned.getId())); - managedQueryOwned.setOwner(user.getId()); - STORAGE.updateExecution(managedQueryOwned); + managedQueryOwned.setOwner(user.getId()); + STORAGE.updateExecution(managedQueryOwned); - // Created not owned execution - final ManagedQuery managedQueryNotOwned = createManagedQuery(); - // Setup user - user.addPermission(ExecutionPermission.onInstance(Ability.READ, managedQueryNotOwned.getId())); + // Created not owned execution + final ManagedQuery managedQueryNotOwned = createManagedQuery(); + // Setup user + user.addPermission(ExecutionPermission.onInstance(Ability.READ, managedQueryNotOwned.getId())); - // Set owner - managedQueryNotOwned.setOwner(user2.getId()); - STORAGE.updateExecution(managedQueryNotOwned); + // Set owner + managedQueryNotOwned.setOwner(user2.getId()); + STORAGE.updateExecution(managedQueryNotOwned); - deletePermissionsOfOwnedInstances(STORAGE, ExecutionPermission.DOMAIN.toLowerCase(), ManagedExecutionId.Parser.INSTANCE, STORAGE::getExecution); + deletePermissionsOfOwnedInstances( + STORAGE, + ExecutionPermission.DOMAIN.toLowerCase(), + ManagedExecutionId.Parser.INSTANCE, + STORAGE::getExecution); - assertThat(user.getPermissions()).containsOnly(ExecutionPermission.onInstance(Ability.READ, managedQueryNotOwned.getId())); + assertThat(user.getPermissions()).containsOnly( + ExecutionPermission.onInstance(Ability.READ, managedQueryNotOwned.getId())); - } + } -} \ No newline at end of file +} diff --git a/backend/src/test/java/com/bakdata/conquery/tasks/QueryCleanupTaskTest.java b/backend/src/test/java/com/bakdata/conquery/tasks/QueryCleanupTaskTest.java index f9d5974d861..803990a915a 100644 --- a/backend/src/test/java/com/bakdata/conquery/tasks/QueryCleanupTaskTest.java +++ b/backend/src/test/java/com/bakdata/conquery/tasks/QueryCleanupTaskTest.java @@ -1,191 +1,198 @@ -package com.bakdata.conquery.tasks; - -import static org.assertj.core.api.Assertions.assertThat; - -import java.time.Duration; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import com.bakdata.conquery.apiv1.query.ConceptQuery; -import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; -import com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery; -import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.models.execution.ManagedExecution; -import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; -import com.bakdata.conquery.models.identifiable.ids.specific.UserId; -import com.bakdata.conquery.models.query.ManagedQuery; -import com.bakdata.conquery.util.NonPersistentStoreFactory; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.TestInstance.Lifecycle; - -@TestInstance(Lifecycle.PER_CLASS) -class QueryCleanupTaskTest { - private static final MetaStorage STORAGE = new NonPersistentStoreFactory().createMetaStorage(); - - private final Duration queryExpiration = Duration.ofDays(30); - - @AfterEach - public void teardownAfterEach() { - STORAGE.clear(); - } - - @Test - void emptyIsEmpty() { - assertThat(STORAGE.getAllExecutions()).isEmpty(); - } - - @Test - void singleUnnamed() throws Exception { - assertThat(STORAGE.getAllExecutions()).isEmpty(); - - createManagedQuery(); - - new QueryCleanupTask(STORAGE, queryExpiration).execute(Map.of(QueryCleanupTask.EXPIRATION_PARAM, List.of("PT719H")), null); - - assertThat(STORAGE.getAllExecutions()).isEmpty(); - } - - private ManagedQuery createManagedQuery() { - final CQAnd root = new CQAnd(); - root.setChildren(new ArrayList<>()); - - ConceptQuery query = new ConceptQuery(root); - - final ManagedQuery managedQuery = new ManagedQuery(query, new UserId("test"), new DatasetId("test"), STORAGE, null, null); - - managedQuery.setCreationTime(LocalDateTime.now().minus(queryExpiration).minusDays(1)); - - STORAGE.addExecution(managedQuery); - managedQuery.setMetaStorage(STORAGE); - - return managedQuery; - } - - @Test - void singleNamed() throws Exception { - assertThat(STORAGE.getAllExecutions()).isEmpty(); - - final ManagedQuery managedQuery = createManagedQuery(); - - managedQuery.setLabel("test"); - - QueryCleanupTask queryCleanupTask = new QueryCleanupTask(STORAGE, queryExpiration); - queryCleanupTask.execute(Map.of(), null); - - assertThat(STORAGE.getAllExecutions()).containsExactlyInAnyOrder(managedQuery); - } - - @Test - void singleNamedButAutoLabeled() throws Exception { - assertThat(STORAGE.getAllExecutions()).isEmpty(); - - final ManagedQuery managedQuery = createManagedQuery(); - - managedQuery.setLabel("test" + ManagedExecution.AUTO_LABEL_SUFFIX); - - new QueryCleanupTask(STORAGE, queryExpiration).execute(Map.of(), null); - - assertThat(STORAGE.getAllExecutions()).isEmpty(); - } - - @Test - void reusedNoNames() throws Exception { - assertThat(STORAGE.getAllExecutions()).isEmpty(); - - final ManagedQuery managedQuery = createManagedQuery(); - final ManagedQuery managedQueryReused = createManagedQuery(); - - managedQuery.setQuery(new ConceptQuery(new CQReusedQuery(managedQueryReused.getId()))); - - new QueryCleanupTask(STORAGE, queryExpiration).execute(Map.of(), null); - - assertThat(STORAGE.getAllExecutions()).isEmpty(); - } - - @Test - void reusedBothNames() throws Exception { - assertThat(STORAGE.getAllExecutions()).isEmpty(); - - final ManagedQuery managedQuery = createManagedQuery(); - managedQuery.setLabel("test1"); - - final ManagedQuery managedQueryReused = createManagedQuery(); - managedQueryReused.setLabel("test2"); - - managedQuery.setQuery(new ConceptQuery(new CQReusedQuery(managedQueryReused.getId()))); - - new QueryCleanupTask(STORAGE, queryExpiration).execute(Map.of(), null); - - assertThat(STORAGE.getAllExecutions()) - .containsExactlyInAnyOrder(managedQuery, managedQueryReused); - } - - @Test - void reusedNames() throws Exception { - assertThat(STORAGE.getAllExecutions()).isEmpty(); - - final ManagedQuery managedQuery = createManagedQuery(); - - final ManagedQuery managedQueryReused = createManagedQuery(); - managedQueryReused.setLabel("test2"); - - managedQuery.setQuery(new ConceptQuery(new CQReusedQuery(managedQueryReused.getId()))); - - new QueryCleanupTask(STORAGE, queryExpiration).execute(Map.of(), null); - - assertThat(STORAGE.getAllExecutions()).containsExactlyInAnyOrder(managedQueryReused); - } - - @Test - void reusedOtherName() throws Exception { - assertThat(STORAGE.getAllExecutions()).isEmpty(); - - final ManagedQuery managedQuery = createManagedQuery(); - managedQuery.setLabel("test2"); - - final ManagedQuery managedQueryReused = createManagedQuery(); - - managedQuery.setQuery(new ConceptQuery(new CQReusedQuery(managedQueryReused.getId()))); - - new QueryCleanupTask(STORAGE, queryExpiration).execute(Map.of(), null); - - assertThat(STORAGE.getAllExecutions()).containsExactlyInAnyOrder(managedQueryReused, managedQuery); - } - - @Test - void reusedTagged() throws Exception { - assertThat(STORAGE.getAllExecutions()).isEmpty(); - - final ManagedQuery managedQuery = createManagedQuery(); - - final ManagedQuery managedQueryReused = createManagedQuery(); - managedQueryReused.setTags(new String[] { "tag" }); - - managedQuery.setQuery(new ConceptQuery(new CQReusedQuery(managedQueryReused.getId()))); - - new QueryCleanupTask(STORAGE, queryExpiration).execute(Map.of(), null); - - assertThat(STORAGE.getAllExecutions()).containsExactlyInAnyOrder(managedQueryReused); - } - - @Test - void reusedYoung() throws Exception { - assertThat(STORAGE.getAllExecutions()).isEmpty(); - - final ManagedQuery managedQuery = createManagedQuery(); - - final ManagedQuery managedQueryReused = createManagedQuery(); - managedQueryReused.setCreationTime(LocalDateTime.now()); - - managedQuery.setQuery(new ConceptQuery(new CQReusedQuery(managedQueryReused.getId()))); - - new QueryCleanupTask(STORAGE, queryExpiration).execute(Map.of(), null); - - assertThat(STORAGE.getAllExecutions()).containsExactlyInAnyOrder(managedQueryReused); - } - -} \ No newline at end of file +package com.bakdata.conquery.tasks; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import com.bakdata.conquery.apiv1.query.ConceptQuery; +import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; +import com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery; +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.models.execution.ManagedExecution; +import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; +import com.bakdata.conquery.models.identifiable.ids.specific.UserId; +import com.bakdata.conquery.models.query.ManagedQuery; +import com.bakdata.conquery.util.NonPersistentStoreFactory; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestInstance.Lifecycle; + +@TestInstance(Lifecycle.PER_CLASS) +class QueryCleanupTaskTest { + private static final MetaStorage STORAGE = new NonPersistentStoreFactory().createMetaStorage(); + + private final Duration queryExpiration = Duration.ofDays(30); + + @AfterEach + public void teardownAfterEach() { + STORAGE.clear(); + } + + @Test + void emptyIsEmpty() { + assertThat(STORAGE.getAllExecutions()).isEmpty(); + } + + @Test + void singleUnnamed() throws Exception { + assertThat(STORAGE.getAllExecutions()).isEmpty(); + + createManagedQuery(); + + new QueryCleanupTask(STORAGE, queryExpiration).execute( + Map.of(QueryCleanupTask.EXPIRATION_PARAM, List.of("PT719H")), + null); + + assertThat(STORAGE.getAllExecutions()).isEmpty(); + } + + private ManagedQuery createManagedQuery() { + final CQAnd root = new CQAnd(); + root.setChildren(new ArrayList<>()); + + ConceptQuery query = new ConceptQuery(root); + + final ManagedQuery managedQuery = new ManagedQuery( + query, + new UserId("test"), + new DatasetId("test"), + STORAGE, + null, + null); + + managedQuery.setCreationTime(LocalDateTime.now().minus(queryExpiration).minusDays(1)); + + STORAGE.addExecution(managedQuery); + managedQuery.setMetaStorage(STORAGE); + + return managedQuery; + } + + @Test + void singleNamed() throws Exception { + assertThat(STORAGE.getAllExecutions()).isEmpty(); + + final ManagedQuery managedQuery = createManagedQuery(); + + managedQuery.setLabel("test"); + + QueryCleanupTask queryCleanupTask = new QueryCleanupTask(STORAGE, queryExpiration); + queryCleanupTask.execute(Map.of(), null); + + assertThat(STORAGE.getAllExecutions()).containsExactlyInAnyOrder(managedQuery); + } + + @Test + void singleNamedButAutoLabeled() throws Exception { + assertThat(STORAGE.getAllExecutions()).isEmpty(); + + final ManagedQuery managedQuery = createManagedQuery(); + + managedQuery.setLabel("test" + ManagedExecution.AUTO_LABEL_SUFFIX); + + new QueryCleanupTask(STORAGE, queryExpiration).execute(Map.of(), null); + + assertThat(STORAGE.getAllExecutions()).isEmpty(); + } + + @Test + void reusedNoNames() throws Exception { + assertThat(STORAGE.getAllExecutions()).isEmpty(); + + final ManagedQuery managedQuery = createManagedQuery(); + final ManagedQuery managedQueryReused = createManagedQuery(); + + managedQuery.setQuery(new ConceptQuery(new CQReusedQuery(managedQueryReused.getId()))); + + new QueryCleanupTask(STORAGE, queryExpiration).execute(Map.of(), null); + + assertThat(STORAGE.getAllExecutions()).isEmpty(); + } + + @Test + void reusedBothNames() throws Exception { + assertThat(STORAGE.getAllExecutions()).isEmpty(); + + final ManagedQuery managedQuery = createManagedQuery(); + managedQuery.setLabel("test1"); + + final ManagedQuery managedQueryReused = createManagedQuery(); + managedQueryReused.setLabel("test2"); + + managedQuery.setQuery(new ConceptQuery(new CQReusedQuery(managedQueryReused.getId()))); + + new QueryCleanupTask(STORAGE, queryExpiration).execute(Map.of(), null); + + assertThat(STORAGE.getAllExecutions()).containsExactlyInAnyOrder(managedQuery, managedQueryReused); + } + + @Test + void reusedNames() throws Exception { + assertThat(STORAGE.getAllExecutions()).isEmpty(); + + final ManagedQuery managedQuery = createManagedQuery(); + + final ManagedQuery managedQueryReused = createManagedQuery(); + managedQueryReused.setLabel("test2"); + + managedQuery.setQuery(new ConceptQuery(new CQReusedQuery(managedQueryReused.getId()))); + + new QueryCleanupTask(STORAGE, queryExpiration).execute(Map.of(), null); + + assertThat(STORAGE.getAllExecutions()).containsExactlyInAnyOrder(managedQueryReused); + } + + @Test + void reusedOtherName() throws Exception { + assertThat(STORAGE.getAllExecutions()).isEmpty(); + + final ManagedQuery managedQuery = createManagedQuery(); + managedQuery.setLabel("test2"); + + final ManagedQuery managedQueryReused = createManagedQuery(); + + managedQuery.setQuery(new ConceptQuery(new CQReusedQuery(managedQueryReused.getId()))); + + new QueryCleanupTask(STORAGE, queryExpiration).execute(Map.of(), null); + + assertThat(STORAGE.getAllExecutions()).containsExactlyInAnyOrder(managedQueryReused, managedQuery); + } + + @Test + void reusedTagged() throws Exception { + assertThat(STORAGE.getAllExecutions()).isEmpty(); + + final ManagedQuery managedQuery = createManagedQuery(); + + final ManagedQuery managedQueryReused = createManagedQuery(); + managedQueryReused.setTags(new String[]{"tag"}); + + managedQuery.setQuery(new ConceptQuery(new CQReusedQuery(managedQueryReused.getId()))); + + new QueryCleanupTask(STORAGE, queryExpiration).execute(Map.of(), null); + + assertThat(STORAGE.getAllExecutions()).containsExactlyInAnyOrder(managedQueryReused); + } + + @Test + void reusedYoung() throws Exception { + assertThat(STORAGE.getAllExecutions()).isEmpty(); + + final ManagedQuery managedQuery = createManagedQuery(); + + final ManagedQuery managedQueryReused = createManagedQuery(); + managedQueryReused.setCreationTime(LocalDateTime.now()); + + managedQuery.setQuery(new ConceptQuery(new CQReusedQuery(managedQueryReused.getId()))); + + new QueryCleanupTask(STORAGE, queryExpiration).execute(Map.of(), null); + + assertThat(STORAGE.getAllExecutions()).containsExactlyInAnyOrder(managedQueryReused); + } + +} diff --git a/backend/src/test/java/com/bakdata/conquery/util/ConqueryEscapeTest.java b/backend/src/test/java/com/bakdata/conquery/util/ConqueryEscapeTest.java index dd2b7c1961b..a60505cce3f 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/ConqueryEscapeTest.java +++ b/backend/src/test/java/com/bakdata/conquery/util/ConqueryEscapeTest.java @@ -5,7 +5,6 @@ import ch.qos.logback.classic.Logger; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.read.ListAppender; - import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; @@ -14,25 +13,10 @@ public class ConqueryEscapeTest { @ParameterizedTest - @CsvSource(value = { - "hallo,hallo", - "test,test", - "test7,test7", - "7test,7test", - "$,$24", - "$24,$2424", - "PO!\"$%&/()=ß,PO$21$22$24$25$26$2f$28$29$3d$c3$9f", - "😈,$f0$9f$98$88", - "aa.aa,aa$2eaa", - "a_a,a_a", - "a-a,a-a", - "a/a,a$2fa", - "a a,a$20a", - "`az{,$60az$7b", // Range border characters - "@AZ[,$40AZ$5b", - "/09:,$2f09$3a", - "test@example.org,test$40example$2eorg" - }) + @CsvSource( + value = {"hallo,hallo", "test,test", "test7,test7", "7test,7test", "$,$24", "$24,$2424", "PO!\"$%&/()=ß,PO$21$22$24$25$26$2f$28$29$3d$c3$9f", "😈,$f0$9f$98$88", "aa.aa,aa$2eaa", "a_a,a_a", "a-a,a-a", "a/a,a$2fa", "a a,a$20a", "`az{,$60az$7b", // Range border characters + "@AZ[,$40AZ$5b", "/09:,$2f09$3a", "test@example.org,test$40example$2eorg" + }) public void testEscaping(String in, String expectedEscaped) { String escaped1 = ConqueryEscape.escape(in); assertThat(escaped1).isEqualTo(expectedEscaped); @@ -61,17 +45,15 @@ public void testEscapingWarningUnescaped() { try { String ignore = ConqueryEscape.unescape("test@example$2eorg"); - assertThat(listAppender.list.getFirst()) - .extracting( - ILoggingEvent::getLevel, - ILoggingEvent::getFormattedMessage - ) - .containsExactly( - ch.qos.logback.classic.Level.WARN, - "Unescaped character '64' at 4 in 'test@example$2eorg'" - ); - } - finally { + assertThat(listAppender.list.getFirst()).extracting( + ILoggingEvent::getLevel, + ILoggingEvent::getFormattedMessage + ) + .containsExactly( + ch.qos.logback.classic.Level.WARN, + "Unescaped character '64' at 4 in 'test@example$2eorg'" + ); + } finally { // Cleanup appender conqueryEscapeLog.detachAppender(listAppender); listAppender.stop(); diff --git a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStore.java b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStore.java index e03263ca918..dffe2532919 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStore.java +++ b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStore.java @@ -1,111 +1,111 @@ -package com.bakdata.conquery.util; - -import java.io.IOException; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.stream.Stream; - -import com.bakdata.conquery.io.storage.Store; -import com.bakdata.conquery.io.storage.xodus.stores.SerializingStore; -import lombok.ToString; - -@ToString -public class NonPersistentStore implements Store { - - private final ConcurrentMap map = new ConcurrentHashMap<>(); - - @Override - public boolean add(KEY key, VALUE value) { - synchronized (map) { - boolean notPresent = !map.containsKey(key); - if(notPresent) { - map.put(key, value); - } - // Was not present before - return notPresent; - } - } - - @Override - public VALUE get(KEY key) { - return map.get(key); - } - - @Override - public SerializingStore.IterationStatistic forEach(StoreEntryConsumer consumer) { - final SerializingStore.IterationStatistic stats = new SerializingStore.IterationStatistic(); - map.forEach((key, value) -> { - consumer.accept(key, value, stats.getTotalProcessed()); - stats.incrTotalProcessed(); - - }); - return stats; - } - - @Override - public boolean update(KEY key, VALUE value) { - map.put(key, value); - return true; - } - - @Override - public boolean remove(KEY key) { - VALUE remove = map.remove(key); - return remove != null; - } - - @Override - public boolean hasKey(KEY key) { - return map.containsKey(key); - } - - @Override - public int count() { - return map.size(); - } - - @Override - public Stream getAll() { - return map.values().stream(); - } - - @Override - public Stream getAllKeys() { - return map.keySet().stream(); - } - - @Override - public void loadData() { - - } - - @Override - public void close() throws IOException { - // Nothing to close - } - - @Override - public void removeStore() { - clear(); - } - - @Override - public void clear() { - map.clear(); - } - - @Override - public String getName() { - return this.getClass().getSimpleName(); - } - - @Override - public void invalidateCache() { - /* Do nothing (semantically this is not a cache, although we hold everything in memory) */ - } - - @Override - public void loadKeys() { - - } -} +package com.bakdata.conquery.util; + +import java.io.IOException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.stream.Stream; + +import com.bakdata.conquery.io.storage.Store; +import com.bakdata.conquery.io.storage.xodus.stores.SerializingStore; +import lombok.ToString; + +@ToString +public class NonPersistentStore implements Store { + + private final ConcurrentMap map = new ConcurrentHashMap<>(); + + @Override + public boolean add(KEY key, VALUE value) { + synchronized (map) { + boolean notPresent = !map.containsKey(key); + if (notPresent) { + map.put(key, value); + } + // Was not present before + return notPresent; + } + } + + @Override + public VALUE get(KEY key) { + return map.get(key); + } + + @Override + public SerializingStore.IterationStatistic forEach(StoreEntryConsumer consumer) { + final SerializingStore.IterationStatistic stats = new SerializingStore.IterationStatistic(); + map.forEach((key, value) -> { + consumer.accept(key, value, stats.getTotalProcessed()); + stats.incrTotalProcessed(); + + }); + return stats; + } + + @Override + public boolean update(KEY key, VALUE value) { + map.put(key, value); + return true; + } + + @Override + public boolean remove(KEY key) { + VALUE remove = map.remove(key); + return remove != null; + } + + @Override + public boolean hasKey(KEY key) { + return map.containsKey(key); + } + + @Override + public int count() { + return map.size(); + } + + @Override + public Stream getAll() { + return map.values().stream(); + } + + @Override + public Stream getAllKeys() { + return map.keySet().stream(); + } + + @Override + public void loadData() { + + } + + @Override + public void close() throws IOException { + // Nothing to close + } + + @Override + public void removeStore() { + clear(); + } + + @Override + public void clear() { + map.clear(); + } + + @Override + public String getName() { + return this.getClass().getSimpleName(); + } + + @Override + public void invalidateCache() { + /* Do nothing (semantically this is not a cache, although we hold everything in memory) */ + } + + @Override + public void loadKeys() { + + } +} diff --git a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java index 6a7c2be26b5..10dfcfab091 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java +++ b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java @@ -88,8 +88,11 @@ public SingletonStore createDatasetStore(String pathName, ObjectMapper } @Override - public IdentifiableStore createSecondaryIdDescriptionStore(String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(secondaryIdDescriptionStore.computeIfAbsent(pathName, n -> new NonPersistentStore<>())); + public IdentifiableStore createSecondaryIdDescriptionStore( + String pathName, + ObjectMapper objectMapper) { + return StoreMappings.identifiable( + secondaryIdDescriptionStore.computeIfAbsent(pathName, n -> new NonPersistentStore<>())); } @Override @@ -163,8 +166,11 @@ public IdentifiableStore createGroupStore(String pathName, ObjectMapper o } @Override - public IdentifiableStore createInternToExternMappingStore(String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(internToExternStore.computeIfAbsent(pathName, n -> new NonPersistentStore<>())); + public IdentifiableStore createInternToExternMappingStore( + String pathName, + ObjectMapper objectMapper) { + return StoreMappings.identifiable( + internToExternStore.computeIfAbsent(pathName, n -> new NonPersistentStore<>())); } @Override diff --git a/backend/src/test/java/com/bakdata/conquery/util/PasswordHelperTest.java b/backend/src/test/java/com/bakdata/conquery/util/PasswordHelperTest.java index b6ae79e8c91..be4cf220ece 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/PasswordHelperTest.java +++ b/backend/src/test/java/com/bakdata/conquery/util/PasswordHelperTest.java @@ -38,10 +38,18 @@ public class PasswordHelperTest { */ static Stream arguments() { return Stream.of( - Arguments.arguments("$argon2id$v=19$m=15360,t=2,p=1$r35m/UGz8lq4ICjcNkb2GUcfYub07450QRTTapYwiJCQDOI9Maa0dlym/iL0AceTNNXgaxLUyGB5EfJoqr+Wng$WVnHZU8uwvufgWPlVh5T+MnTtX5Ry0hhCD0ej90L0Kk", Argon2Function.getInstance(15360, 2, 1, 32, Argon2.ID)), - Arguments.arguments("$100801$SBpPHCtLT+2FbJ2BS49J4sgRXfvduVm17U9yd0Ygky/3MgUgK1r4LMixKSQX4LQjSEuE6tV8ibABXXAr9tCZKA==$aPTssj2maVw34QgrhRIsUHu6irB1NrjiFpdpUXFHHA+XhjPG03PKrbj5CBXJx3cCUosU/IARQliSW2LWRLFtiw==", ScryptFunction.getInstance(65536, 8, 1, 64)), - Arguments.arguments("$2b$10$YMPj.MoAs81tO8HzrCYxnOujaPwbu5SGsSrdNyxdIJ9BlBIv9i0t.", BcryptFunction.getInstance(Bcrypt.B, 10)), - Arguments.arguments("$3$1331439861760256$+Rqke26gKhtP60UkVR2a3SfszrOkVrMiJ6LZUWvl2vI5OpW815zKiod8Sdz3aOcuajo6c1iKEXcWjk61emmgTw==$CiH0mwqibUZD5R5HqFNpaYCkWjiYcTQe0sjG+4ZYw/A=", CompressedPBKDF2Function.getInstance(Hmac.SHA256, 310000, 256)) + Arguments.arguments( + "$argon2id$v=19$m=15360,t=2,p=1$r35m/UGz8lq4ICjcNkb2GUcfYub07450QRTTapYwiJCQDOI9Maa0dlym/iL0AceTNNXgaxLUyGB5EfJoqr+Wng$WVnHZU8uwvufgWPlVh5T+MnTtX5Ry0hhCD0ej90L0Kk", + Argon2Function.getInstance(15360, 2, 1, 32, Argon2.ID)), + Arguments.arguments( + "$100801$SBpPHCtLT+2FbJ2BS49J4sgRXfvduVm17U9yd0Ygky/3MgUgK1r4LMixKSQX4LQjSEuE6tV8ibABXXAr9tCZKA==$aPTssj2maVw34QgrhRIsUHu6irB1NrjiFpdpUXFHHA+XhjPG03PKrbj5CBXJx3cCUosU/IARQliSW2LWRLFtiw==", + ScryptFunction.getInstance(65536, 8, 1, 64)), + Arguments.arguments( + "$2b$10$YMPj.MoAs81tO8HzrCYxnOujaPwbu5SGsSrdNyxdIJ9BlBIv9i0t.", + BcryptFunction.getInstance(Bcrypt.B, 10)), + Arguments.arguments( + "$3$1331439861760256$+Rqke26gKhtP60UkVR2a3SfszrOkVrMiJ6LZUWvl2vI5OpW815zKiod8Sdz3aOcuajo6c1iKEXcWjk61emmgTw==$CiH0mwqibUZD5R5HqFNpaYCkWjiYcTQe0sjG+4ZYw/A=", + CompressedPBKDF2Function.getInstance(Hmac.SHA256, 310000, 256)) ); } diff --git a/backend/src/test/java/com/bakdata/conquery/util/extensions/GroupExtension.java b/backend/src/test/java/com/bakdata/conquery/util/extensions/GroupExtension.java index 8c0c2a890e9..87061ac0c20 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/extensions/GroupExtension.java +++ b/backend/src/test/java/com/bakdata/conquery/util/extensions/GroupExtension.java @@ -15,6 +15,7 @@ public GroupExtension(MetaStorage metaStorage, String name) { group = new Group(name, name, metaStorage); } + @Override public void beforeAll(ExtensionContext context) throws Exception { group.updateStorage(); diff --git a/backend/src/test/java/com/bakdata/conquery/util/extensions/MockServerExtension.java b/backend/src/test/java/com/bakdata/conquery/util/extensions/MockServerExtension.java index 8315c044d46..24048bf3767 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/extensions/MockServerExtension.java +++ b/backend/src/test/java/com/bakdata/conquery/util/extensions/MockServerExtension.java @@ -26,7 +26,7 @@ public void afterAll(ExtensionContext context) throws Exception { server.stop(); } - public String baseUrl(){ + public String baseUrl() { return "http://localhost:%d".formatted(server.getPort()); } } diff --git a/backend/src/test/java/com/bakdata/conquery/util/extensions/NamespaceStorageExtension.java b/backend/src/test/java/com/bakdata/conquery/util/extensions/NamespaceStorageExtension.java index b01d28f2ca8..feac4380c64 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/extensions/NamespaceStorageExtension.java +++ b/backend/src/test/java/com/bakdata/conquery/util/extensions/NamespaceStorageExtension.java @@ -22,4 +22,4 @@ public void beforeAll(ExtensionContext context) throws Exception { public void beforeEach(ExtensionContext context) { storage.clear(); } -} \ No newline at end of file +} diff --git a/backend/src/test/java/com/bakdata/conquery/util/extensions/SolrServerExtension.java b/backend/src/test/java/com/bakdata/conquery/util/extensions/SolrServerExtension.java index 769bd544923..e7431ee20c8 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/extensions/SolrServerExtension.java +++ b/backend/src/test/java/com/bakdata/conquery/util/extensions/SolrServerExtension.java @@ -27,14 +27,17 @@ public void beforeAll(ExtensionContext context) throws Exception { solrBaseUrl = System.getenv(SOLR_BASE_URL_ENV); if (solrBaseUrl != null) { - log.info("Environment variable {} was set to {}. Using external solr for testing.", SOLR_BASE_URL_ENV, solrBaseUrl); + log.info( + "Environment variable {} was set to {}. Using external solr for testing.", + SOLR_BASE_URL_ENV, + solrBaseUrl); return; } log.info("Spin up local container"); solrContainer = new SolrContainer(DockerImageName.parse("solr:9.10.1")); - solrContainer.withCollection(collection); - solrContainer.start(); + solrContainer.withCollection(collection); + solrContainer.start(); solrBaseUrl = "http://" + solrContainer.getHost() + ":" + solrContainer.getSolrPort() + "/solr"; } diff --git a/backend/src/test/java/com/bakdata/conquery/util/extensions/XodusEnvironmentExtension.java b/backend/src/test/java/com/bakdata/conquery/util/extensions/XodusEnvironmentExtension.java index 7a0678425b3..6407727b1b9 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/extensions/XodusEnvironmentExtension.java +++ b/backend/src/test/java/com/bakdata/conquery/util/extensions/XodusEnvironmentExtension.java @@ -31,7 +31,7 @@ public void beforeAll(ExtensionContext context) throws Exception { @Override public void afterAll(ExtensionContext context) throws Exception { environment.close(); - if(!envPath.delete()) { + if (!envPath.delete()) { log.warn("Could not delete test directory: {}", envPath); } } diff --git a/backend/src/test/java/com/bakdata/conquery/util/progressreporter/ProgressReporterTest.java b/backend/src/test/java/com/bakdata/conquery/util/progressreporter/ProgressReporterTest.java index 8f240f9b1ef..8d7112a003b 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/progressreporter/ProgressReporterTest.java +++ b/backend/src/test/java/com/bakdata/conquery/util/progressreporter/ProgressReporterTest.java @@ -18,9 +18,9 @@ public class ProgressReporterTest { private List allOccurencesOf(String haystack, String needle) { List occurences = new ArrayList<>(); int lastIndex = 0; - while(lastIndex != -1) { - lastIndex = haystack.indexOf(needle,lastIndex); - if(lastIndex != -1){ + while (lastIndex != -1) { + lastIndex = haystack.indexOf(needle, lastIndex); + if (lastIndex != -1) { occurences.add(lastIndex); lastIndex += 1; } @@ -30,9 +30,9 @@ private List allOccurencesOf(String haystack, String needle) { @Test public void alignmentTest() throws InterruptedException { - ProgressReporterImpl pr = (ProgressReporterImpl)ProgressReporter.createWaiting(); + ProgressReporterImpl pr = (ProgressReporterImpl) ProgressReporter.createWaiting(); Thread.sleep(2_000); - + pr.start(); pr.setMax(100); pr.report(1); @@ -44,41 +44,43 @@ public void alignmentTest() throws InterruptedException { Thread.sleep(100); assertThat(occurenceOfHour).isEqualTo(allOccurencesOf(pr.getEstimate(), "h")); } - + } - + @Test public void basicTest() throws InterruptedException { - + ProgressReporter pr = ProgressReporter.createStarted(); pr.setMax(100); assertThat(pr.getEstimate()).contains(UNKNOWN); Thread.sleep(100); pr.report(1); - + log.info(pr.getEstimate()); pr.report(99); assertThat(pr.isDone()).isFalse(); pr.done(); - + assertThat(pr.isDone()).isTrue(); assertThat(pr.getEstimate()).isEqualTo(MAX_PROGRESS); } - + @Test public void serialisationTest() throws JsonProcessingException, InterruptedException { - ProgressReporterImpl pr = (ProgressReporterImpl)ProgressReporter.createStarted(); + ProgressReporterImpl pr = (ProgressReporterImpl) ProgressReporter.createStarted(); pr.setMax(100); assertThat(pr.getEstimate()).contains(UNKNOWN); Thread.sleep(100); pr.report(1); log.info(pr.getEstimate()); - + JsonNode json = Jackson.MAPPER.valueToTree(pr); log.info(json.asText()); - ImmutableProgressReporter deserialized = (ImmutableProgressReporter) Jackson.MAPPER.treeToValue(json, ProgressReporter.class); - + ImmutableProgressReporter deserialized = (ImmutableProgressReporter) Jackson.MAPPER.treeToValue( + json, + ProgressReporter.class); + log.info(deserialized.getEstimate()); Thread.sleep(100); log.info(deserialized.getEstimate()); diff --git a/backend/src/test/java/com/bakdata/conquery/util/progressreporter/ProgressReporterUtilTest.java b/backend/src/test/java/com/bakdata/conquery/util/progressreporter/ProgressReporterUtilTest.java index d03dcacb46e..1f3a0cf77dc 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/progressreporter/ProgressReporterUtilTest.java +++ b/backend/src/test/java/com/bakdata/conquery/util/progressreporter/ProgressReporterUtilTest.java @@ -12,7 +12,11 @@ class ProgressReporterUtilTest { @Test void buildProgressReportString() { - assertThat(ProgressReporterUtil.buildProgressReportString(false, 20, 100, TimeUnit.MINUTES.toMillis(2),0)).endsWith("est. 00h 08m 00s "); - assertThat(ProgressReporterUtil.buildProgressReportString(false, 1, 10, TimeUnit.MINUTES.toMillis(2),0)).endsWith("est. 00h 18m 00s "); + assertThat( + ProgressReporterUtil.buildProgressReportString(false, 20, 100, TimeUnit.MINUTES.toMillis(2), 0)).endsWith( + "est. 00h 08m 00s "); + assertThat( + ProgressReporterUtil.buildProgressReportString(false, 1, 10, TimeUnit.MINUTES.toMillis(2), 0)).endsWith( + "est. 00h 18m 00s "); } -} \ No newline at end of file +} diff --git a/backend/src/test/java/com/bakdata/conquery/util/search/QuickSearchTest.java b/backend/src/test/java/com/bakdata/conquery/util/search/QuickSearchTest.java index 3c07fb13fb0..c0df9ca7a2d 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/search/QuickSearchTest.java +++ b/backend/src/test/java/com/bakdata/conquery/util/search/QuickSearchTest.java @@ -14,7 +14,16 @@ class QuickSearchTest { @Test public void anaconda() { final TrieSearch search = new TrieSearch<>(3, ""); - for (String item : List.of("Anaconda", "Anaxonds" /* Simulate Typing Errors */, "Ananas", "Honda", "London", "Analysis", "Canada", "Condor", "Condar")) { + for (String item : List.of( + "Anaconda", + "Anaxonds" /* Simulate Typing Errors */, + "Ananas", + "Honda", + "London", + "Analysis", + "Canada", + "Condor", + "Condar")) { search.addItem(item, List.of(item)); } @@ -23,17 +32,18 @@ public void anaconda() { String query = "anaconda"; final List results = search.findItems(List.of(query), Integer.MAX_VALUE); - assertThat(results) - .describedAs("Query for %s".formatted(query)) - .isEqualTo(List.of("Anaconda", // Full match - "Condar", // 3 trigrams - "Condor", // 2 trigrams - "Honda", // 2 trigrams - "Anaxonds", // 1 trigram - "London", // 1 trigram - "Ananas", // 1 trigram - "Analysis", // 1 trigram - "Canada" // 1 trigram + assertThat(results).describedAs("Query for %s".formatted(query)) + .isEqualTo( + List.of( + "Anaconda", // Full match + "Condar", // 3 trigrams + "Condor", // 2 trigrams + "Honda", // 2 trigrams + "Anaxonds", // 1 trigram + "London", // 1 trigram + "Ananas", // 1 trigram + "Analysis", // 1 trigram + "Canada" // 1 trigram )); } @@ -50,9 +60,9 @@ public void pants() { String query = "pants"; final List results = search.findItems(List.of(query), Integer.MAX_VALUE); - assertThat(results) - .describedAs("Query for %s".formatted(query)) - .isEqualTo(List.of("Pants", "PantsPants", "Pantshop", "Sweatpants", "Rantsom", "Fantastic")); + assertThat(results).describedAs("Query for %s".formatted(query)) + .isEqualTo( + List.of("Pants", "PantsPants", "Pantshop", "Sweatpants", "Rantsom", "Fantastic")); } @Test @@ -62,15 +72,14 @@ public void searchOrder() { // The more hits an item has, the more do we favor it. - assertThat(search.findItems(List.of("aa", "c"), Integer.MAX_VALUE)) - .containsExactly( - "c aa", // Two exact matches - "aa", // One exact match - "c", // One exact match - "aaa", // One prefix match, onto a whole word - "aab", // One prefix match, onto a whole word - "d baaacd" // Two partial matches - ); + assertThat(search.findItems(List.of("aa", "c"), Integer.MAX_VALUE)).containsExactly( + "c aa", // Two exact matches + "aa", // One exact match + "c", // One exact match + "aaa", // One prefix match, onto a whole word + "aab", // One prefix match, onto a whole word + "d baaacd" // Two partial matches + ); // However negative matches are not considered (ie "c" is not used to weigh against "c aa") assertThat(search.findItems(List.of("aa"), 4)).containsExactly("aa", "c aa", "aaa", "aab"); @@ -80,14 +89,14 @@ private static TrieSearch setup() { final TrieSearch search = new TrieSearch<>(2, ""); List items = List.of( - "a", - "aa", - "aaa", - "aab", - "b", - "c", - "c aa", - "d baaacd" + "a", + "aa", + "aaa", + "aab", + "b", + "c", + "c aa", + "d baaacd" ); for (String item : items) { @@ -113,20 +122,17 @@ public void searchIdentities() { public void testNGrams() { final TrieSearch search = new TrieSearch<>(2, null); - assertThat(search.ngramSplit("baaacd")) - .containsExactly( - "ba", - "aa", - "aa", - "ac", - "cd" - ); + assertThat(search.ngramSplit("baaacd")).containsExactly( + "ba", + "aa", + "aa", + "ac", + "cd" + ); - assertThat(search.ngramSplit("acd")) - .containsExactly("ac", "cd"); + assertThat(search.ngramSplit("acd")).containsExactly("ac", "cd"); - assertThat(search.ngramSplit("aacd")) - .containsExactly("aa", "ac", "cd"); + assertThat(search.ngramSplit("aacd")).containsExactly("aa", "ac", "cd"); } @Test @@ -139,4 +145,4 @@ public void testNoNGram() { assertThat(search.ngramSplit("aacd")).isEmpty(); } -} \ No newline at end of file +} diff --git a/backend/src/test/java/com/bakdata/conquery/util/search/SolrFilterValueTest.java b/backend/src/test/java/com/bakdata/conquery/util/search/SolrFilterValueTest.java index 0b6104ee262..9aa7b5f6153 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/search/SolrFilterValueTest.java +++ b/backend/src/test/java/com/bakdata/conquery/util/search/SolrFilterValueTest.java @@ -70,10 +70,17 @@ public class SolrFilterValueTest { public static final Environment ENVIRONMENT = new Environment(SolrFilterValueTest.class.getSimpleName()); public static final ConqueryConfig CONQUERY_CONFIG = new ConqueryConfig(); @RegisterExtension - private static final MockServerExtension REF_SERVER = new MockServerExtension(ClientAndServer.startClientAndServer(), SolrFilterValueTest::initRefServer); + private static final MockServerExtension REF_SERVER = new MockServerExtension( + ClientAndServer.startClientAndServer(), + SolrFilterValueTest::initRefServer); @RegisterExtension private static final SolrServerExtension SOLR_SERVER = new SolrServerExtension(DATASET_ID.toString()); - private static final IndexService INDEX_SERVICE = new IndexService(new CsvParserSettings(){{setDelimiterDetectionEnabled(true);setLineSeparatorDetectionEnabled(true);}}); + private static final IndexService INDEX_SERVICE = new IndexService(new CsvParserSettings() { + { + setDelimiterDetectionEnabled(true); + setLineSeparatorDetectionEnabled(true); + } + }); public static final SelectFilter FILTER = createFilter(); public static SolrConfig solrConfig; public static SolrProcessor searchProcessor; @@ -83,7 +90,11 @@ public static void initRefServer(ClientAndServer mockServer) { try (InputStream inputStream = LoadingUtil.openResource(MAPPING_PATH)) { mockServer.when(request().withPath("/mapping.csv")) - .respond(HttpResponse.response().withContentType(new MediaType("text", "csv")).withBody(inputStream.readAllBytes())); + .respond( + HttpResponse.response() + .withContentType(new MediaType("text", "csv")) + .withBody( + inputStream.readAllBytes())); } } @@ -124,18 +135,26 @@ public String getFilterType() { @SneakyThrows(URISyntaxException.class) @Override public List getSearchReferences() { - LabelMap labelMap = new LabelMap(getId(), ImmutableBiMap.of( - "a", "Map A", - "map b", "Map B", - "map c", "Map C", - "e", "Map E" // exists in all sources - ), 0, false); + LabelMap labelMap = new LabelMap( + getId(), + ImmutableBiMap.of( + "a", + "Map A", + "map b", + "Map B", + "map c", + "Map C", + "e", + "Map E" // exists in all sources + ), + 0, + false); final FilterTemplate index = new FilterTemplate( - new URI("/mapping.csv"), - "internal", - "{{external}}", - "" + new URI("/mapping.csv"), + "internal", + "{{external}}", + "" ); index.setName("test1"); index.setIndexService(INDEX_SERVICE); @@ -152,7 +171,10 @@ public FilterId getId() { }; } - public static Collection findExact(SolrProcessor solrProcessor, SelectFilter filter, String searchTerm) { + public static Collection findExact( + SolrProcessor solrProcessor, + SelectFilter filter, + String searchTerm) { return solrProcessor.findExact(filter, List.of(searchTerm)).resolved(); } @@ -160,12 +182,20 @@ public static Collection findExact(SolrProcessor solrProcessor, S @Order(0) public void addData() throws InterruptedException, SolrServerException, IOException { // Index values from concept/reference - Set managerSearchables = FILTER.getSearchReferences().stream().filter(ref -> !(ref instanceof Column)).collect(Collectors.toSet()); - searchProcessor.indexManagerResidingSearches(managerSearchables, new AtomicBoolean(false), new ProgressReporterImpl()); + Set managerSearchables = FILTER.getSearchReferences() + .stream() + .filter( + ref -> !(ref instanceof Column)) + .collect(Collectors.toSet()); + searchProcessor.indexManagerResidingSearches( + managerSearchables, + new AtomicBoolean(false), + new ProgressReporterImpl()); // Index values from column Column column = createSearchable(); - ArrayList strings = new ArrayList<>(List.of( + ArrayList strings = new ArrayList<>( + List.of( "a", // should be shadowed by LabelMap "b", // should be shadowed by external csv map "column c", @@ -173,7 +203,7 @@ public void addData() throws InterruptedException, SolrServerException, IOExcept "column ab", "column ba", "" // Empty string handling - )); + )); // Null-Handling (adds null explicitly because List.of forbids it) strings.add(null); @@ -221,15 +251,16 @@ public void findEmptyTermFirstPage() { AutoCompleteResult actual = searchProcessor.query(FILTER, "", 5, 0); assertThat(actual).satisfies(uut -> { - assertThat(uut.values()).isEqualTo(List.of( - new FrontendValue("", "No Value", null), - new FrontendValue("a", "Map A", null), - new FrontendValue("e", "Map E", null), - new FrontendValue("map b", "Map B", null), - new FrontendValue("map c", "Map C", null) - )); - assertThat(uut.total()).isEqualTo(14); - } + assertThat(uut.values()).isEqualTo( + List.of( + new FrontendValue("", "No Value", null), + new FrontendValue("a", "Map A", null), + new FrontendValue("e", "Map E", null), + new FrontendValue("map b", "Map B", null), + new FrontendValue("map c", "Map C", null) + )); + assertThat(uut.total()).isEqualTo(14); + } ); } @@ -239,15 +270,16 @@ public void findEmptyTermSecondPage() { AutoCompleteResult actual = searchProcessor.query(FILTER, "", 5, 1); assertThat(actual).satisfies(uut -> { - assertThat(uut.values()).isEqualTo(List.of( - new FrontendValue("b", "Data b", "b"), - new FrontendValue("data a", "data a", "data a"), - new FrontendValue("data c", "Data C", "data c"), - new FrontendValue("data d", "data d", "data d"), - new FrontendValue("external-null", "external-null", "external-null") - )); - assertThat(uut.total()).isEqualTo(14); - } + assertThat(uut.values()).isEqualTo( + List.of( + new FrontendValue("b", "Data b", "b"), + new FrontendValue("data a", "data a", "data a"), + new FrontendValue("data c", "Data C", "data c"), + new FrontendValue("data d", "data d", "data d"), + new FrontendValue("external-null", "external-null", "external-null") + )); + assertThat(uut.total()).isEqualTo(14); + } ); } @@ -257,14 +289,15 @@ public void findEmptyTermThirdPage() { AutoCompleteResult actual = searchProcessor.query(FILTER, "", 5, 2); assertThat(actual).satisfies(uut -> { - assertThat(uut.values()).isEqualTo(List.of( - new FrontendValue("","internal", null), - new FrontendValue("column ab", "column ab", "null"), - new FrontendValue("column ba", "column ba", "null"), - new FrontendValue("column c", "column c", "null") - )); - assertThat(uut.total()).isEqualTo(14); - } + assertThat(uut.values()).isEqualTo( + List.of( + new FrontendValue("", "internal", null), + new FrontendValue("column ab", "column ab", "null"), + new FrontendValue("column ba", "column ba", "null"), + new FrontendValue("column c", "column c", "null") + )); + assertThat(uut.total()).isEqualTo(14); + } ); } @@ -274,24 +307,24 @@ public void findTerm1() { AutoCompleteResult actual = searchProcessor.query(FILTER, "a", 25, 0); assertThat(actual).isEqualTo( - new AutoCompleteResult( - List.of( - new FrontendValue("a", "Map A", "null"), - new FrontendValue("data a", "Data", "data a"), - new FrontendValue("map b", "Map B", "null"), - new FrontendValue("map c", "Map C", "null"), - new FrontendValue("e", "Map E", "null"), - new FrontendValue("b", "Data b", "b"), - new FrontendValue("data c", "Data C", "data c"), - new FrontendValue("data d", "data d", "data d"), - new FrontendValue("", "internal", null), - new FrontendValue("external-null", "external-null", "external-null"), - new FrontendValue("column ab", "column ab", "null"), - new FrontendValue("column ba", "column ba", "null"), - new FrontendValue("column c", "column c", "null") - ), - 13 - ) + new AutoCompleteResult( + List.of( + new FrontendValue("a", "Map A", "null"), + new FrontendValue("data a", "Data", "data a"), + new FrontendValue("map b", "Map B", "null"), + new FrontendValue("map c", "Map C", "null"), + new FrontendValue("e", "Map E", "null"), + new FrontendValue("b", "Data b", "b"), + new FrontendValue("data c", "Data C", "data c"), + new FrontendValue("data d", "data d", "data d"), + new FrontendValue("", "internal", null), + new FrontendValue("external-null", "external-null", "external-null"), + new FrontendValue("column ab", "column ab", "null"), + new FrontendValue("column ba", "column ba", "null"), + new FrontendValue("column c", "column c", "null") + ), + 13 + ) ); } @@ -301,22 +334,22 @@ public void findTerm2() { AutoCompleteResult actual = searchProcessor.query(FILTER, "ab", 25, 0); assertThat(actual).isEqualTo( - new AutoCompleteResult( - List.of( - new FrontendValue("column ab", "column ab", null), - new FrontendValue("column ba", "column ba", null), - new FrontendValue("a", "Map A", null), - new FrontendValue("map b", "Map B", null), - new FrontendValue("map c", "Map C", null), - new FrontendValue("e", "Map E", "null"), - new FrontendValue("data a", "Data", "data a"), - new FrontendValue("b", "Data B", "b"), - new FrontendValue("data c", "Data C", "data c"), - new FrontendValue("data d", "data d", "data d"), - new FrontendValue("column c", "column c", null) - ), - 11 - ) + new AutoCompleteResult( + List.of( + new FrontendValue("column ab", "column ab", null), + new FrontendValue("column ba", "column ba", null), + new FrontendValue("a", "Map A", null), + new FrontendValue("map b", "Map B", null), + new FrontendValue("map c", "Map C", null), + new FrontendValue("e", "Map E", "null"), + new FrontendValue("data a", "Data", "data a"), + new FrontendValue("b", "Data B", "b"), + new FrontendValue("data c", "Data C", "data c"), + new FrontendValue("data d", "data d", "data d"), + new FrontendValue("column c", "column c", null) + ), + 11 + ) ); } @@ -326,14 +359,14 @@ public void findPhrase1() { AutoCompleteResult actual = searchProcessor.query(FILTER, "column a", 25, 0); assertThat(actual).isEqualTo( - new AutoCompleteResult( - List.of( - new FrontendValue("column ab", "column ab", null), - new FrontendValue("column ba", "column ba", null), - new FrontendValue("column c", "column c", null) - ), - 3 - ) + new AutoCompleteResult( + List.of( + new FrontendValue("column ab", "column ab", null), + new FrontendValue("column ba", "column ba", null), + new FrontendValue("column c", "column c", null) + ), + 3 + ) ); } @@ -343,13 +376,13 @@ public void findPhrase1LimitPage0() { AutoCompleteResult actual = searchProcessor.query(FILTER, "column a", 2, 0); assertThat(actual).isEqualTo( - new AutoCompleteResult( - List.of( - new FrontendValue("column ab", "column ab", null), - new FrontendValue("column ba", "column ba", null) - ), - 3 - ) + new AutoCompleteResult( + List.of( + new FrontendValue("column ab", "column ab", null), + new FrontendValue("column ba", "column ba", null) + ), + 3 + ) ); } @@ -359,13 +392,13 @@ public void findPhrase1LimitPage1() { AutoCompleteResult actual = searchProcessor.query(FILTER, "column a", 2, 1); assertThat(actual).isEqualTo( - new AutoCompleteResult( - List.of( - new FrontendValue("column c", "column c", null) + new AutoCompleteResult( + List.of( + new FrontendValue("column c", "column c", null) - ), - 3 - ) + ), + 3 + ) ); } @@ -400,228 +433,239 @@ public void findExactUppercase() { @Order(3) public void findExactMultiple() { - ConceptsProcessor.ExactFilterValueResult actual = searchProcessor.findExact(FILTER, List.of("MAP A", "z", "Data e")); + ConceptsProcessor.ExactFilterValueResult actual = searchProcessor.findExact( + FILTER, + List.of("MAP A", "z", "Data e")); - assertThat(actual).usingRecursiveComparison().isEqualTo(new ConceptsProcessor.ExactFilterValueResult(List.of(new FrontendValue("a", "Map A"), new FrontendValue("e", "Data E", "e")),Set.of("z"))); + assertThat(actual).usingRecursiveComparison() + .isEqualTo( + new ConceptsProcessor.ExactFilterValueResult( + List.of(new FrontendValue("a", "Map A"), new FrontendValue("e", "Data E", "e")), + Set.of("z"))); } static final Set MANY_VALUES = Set.of( - "RwovlXQ1snDlQPmLcMtX", - "m1QKhB35vjvrw8cWDPIL", - "tu2LJwmqDSRmVKETE7P4", - "ZTppQoULmN7DmAoAGbkX", - "Ljet3MVkeWZEFT4cn8kZ", - "1gjEY4upDUqKEzTteSf8", - "nwVhyW0W8RoqIgeZHWII", - "O1SfaHkMKiAeKlCB3i3X", - "JCZLQOSbHloHSJCG1V4M", - "499eumzmxhxaGkr8UjiV", - "xt1QYhiDPIJvSKDAu9Bm", - "4Q244beSDOPrAR969i18", - "JyRV4qyujIpDGOFP23nk", - "ogGCUNFLKxEZL8Kj2hkR", - "TP2qyxIJO9yUlN2gkI2R", - "3njVrHwWoSz2anaSL91W", - "ekmvmUi7KzLoIKNw4QD6", - "nCP76KQeCofFa4OSxRpM", - "CzPW5YkbnLoY58WLbGXM", - "UxI2wSnBcg4mVTNjZ9Y5", - "rzOfJ52EEWKuPjgvmRJi", - "oGlxr9cwD49ExsXh604B", - "J9EbzvRUcnZwAoSxJhq2", - "tFVAoBwHoHYqmxHoHbWT", - "k2VVuS7T3INCBP6J4Hjo", - "OhDxPFqSNtXxMM9yJdAv", - "qjKIJ5fRfruj2iBua5qd", - "hcqeXnjuq0Qq4E8xn8Ib", - "mGVIPXwXQ444vvLBGvna", - "MYR4uz7mvdVfLaARDN2T", - "kdyc0yE3UdjJ3biajnJj", - "LshtBGUiN6b4X6dVjVrj", - "fCVDq2WvdynfFGp4R0E4", - "ossClcGwq5wLRUF8QN1r", - "9TDxe9FjLo1fAqOMGv0k", - "yK7vrtg9mbtYYE6K9qwa", - "osU8sYsgRa6IKO4xevDw", - "EVtiG3yOFeC32NLKSbcB", - "102WCDM79qgEfC3rdvrZ", - "zKaInP73dDDL3Sj0RnU3", - "3JQp2vezxP5JEA5WaCsG", - "Gjws7sVd16gJQWIbCSE9", - "bW6iZn9iQZPfv4sjHpiy", - "e1FF6FNJ1TVkTK40kK0Y", - "loVi3ZlegdXKt5KuIWD9", - "gbf0a1pg33NBhVfeA9Jd", - "ZxklYSrneuEKWmGr5q0H", - "8pY9zKWRSXYztTtX2Dsu", - "uDTnyMyuZ8dTKXtZfwnD", - "JpLP8mH5d9Bjl8nT7Eot", - "H8sWEwjn5cNP336XRjBm", - "FgGZRHNjiNFSjA9jSNpd", - "JoWsHlySX30ytc4Nrjgp", - "z9EUPGqJJ2psxI4w5eDx", - "THAfYCTS7kdoOwdMbem9", - "tW2dIuFa6xmKk23HW0bB", - "PhZ9g08JZPF4wJkujckI", - "16HEc1MMs08jOHElnffy", - "QhlWwcgkIcgeXWOFLhhE", - "GFmxh1hjvUW7PWimDZ2f", - "BYEfxiWXia1NNjLhUc8q", - "GC3qOEl7VjK7DvQc5J2L", - "Bn9f8P5jRslBtmKiZPzy", - "wPRahJuThxpluEIx81lU", - "9SWXp9YbDFEBlcuVZUaC", - "yLilVeNK1SCG3ASFVbmq", - "FT276Es24XGJhUvk5qRx", - "sjp8ncb9dLkdoL8LLZhv", - "eDkmN2xZ39mHoTNaKNxE", - "8GAe0ewJl0zuYZSjEEXR", - "dYux7hQAPNoptIU2YDW6", - "OF8pMK4BwiSTAwsZdSaz", - "IGJA9flaWsVkVjB3goID", - "RZlMG1eR6ZjTNncj5Oxu", - "7f5RIlXE0l1PWC8SFjet", - "phqN9Aj9RWiJJKVBGoac", - "8CZDNMWZNLavWYudTFzU", - "6vnLzbizEMz6ONGGIXsh", - "yyHYfMpGlDP4jGYHwPJD", - "cZ0C7GKi6fxGLI23aBcp", - "7jspiPcDFpcTI5eoXOa4", - "T54xCtrhiuSutpyoyzm9", - "W9Vo7fweW7TxBeNi0Wkp", - "CJGkWtCwUQ3VRlt0Fozl", - "H7UrsRkQO65anpwDSHhL", - "uAHUDeX0dQH4d5jEBd3t", - "EagFaADXRsIZ1PJk3D2a", - "N2Uk836xTQ34jSyi7WNE", - "3hV3IeIFQoVXy941lzqm", - "ROc9wJQgwY29jQb9MQJs", - "EPi6OLT3hfhLmVRgUChO", - "K4U5dKsvrofd7mY3l5lp", - "I2MNEbvABg7tZG2tqtx2", - "qnxDgpgNrK4mid4QoJLz", - "pTT23LxARdAXVuW0qqCx", - "EkVVOXSSX0yvgcnngYZ6", - "lQShCmtrpJwb8rOtKGMQ", - "pdq1TKljIVxfcakWtpdg", - "KLM0PRS7rMp1SQUeK8c6", - "Ho4uzFCzMwrg7t2Pmoxz", - "xZQDDgfO9HqqAd03nA90", - "kXpETSkGLtGzJhaQ946K", - "xcJwuXgmCoawcGuwJLaE", - "UbO651ujKWbhulALO5LC", - "cmGfI01YjgMgPPyxwajS", - "vhekh5AvXYXt2NeDqJPh", - "jhKi9ZwU0VubFCiGDd2J", - "y2DWsfpdK3FZ8N4FkTAW", - "JYj3T6aQjkYTdGirBWr3", - "cGI2HUgXQwzMMaDhEA6D", - "XD9UuN053Zj5EGgi6ocm", - "M64qxq184jKHqk8vlW8J", - "oo3DkGGKkx57w7ypr8pp", - "UVHNcXJr93Lyo2pjD94F", - "k8GMtUxynpstFQ4FNvWi", - "OyCCPtZ17sYgc9FYLJrS", - "Yz0wD1llAL3a33txe7lX", - "Td0WEyqKEelw5E7wajRD", - "twl0YyWCZ1tDe3P8YvYx", - "js3FbR48v08Lc6FZD29D", - "dqDv6kLFdvvcE4Y5s2oz", - "nafPV1EopdZRibEjxkMU", - "VutMAZoKihv9pu5DNYXe", - "Ee7OFTqUxbapi7sBKKQ5", - "C1wIwOmbwjqTGoNiOEuL", - "EyNbtpcUTQBAHPoaVku1", - "c78cAxCXU8gwbvwcpC7F", - "QxwVWcjHD5hAlcXYw2UU", - "zz3PhVgI0dWvmKEm5VaD", - "GrrpSofZ5n3ZutyPJfo7", - "6DGsMGG2vnupr6zgaAPo", - "QwCXcdq2utJLdqDg8Bt3", - "8SFeow3rt7shOQGkwG9Y", - "lXmRfrosiEGKyqbsyG15", - "jsSoutG1g5hzrzjR2U2s", - "Jnau7OBpvvY6VyQRQnaY", - "RDWZwwFQSGKB5sQ9bGI3", - "C2qFceAcoks7Oqq8RDT1", - "4hKjWW1NKEqWoVAB5g9Q", - "Ia5sNUqzWcUEfQszYVCT", - "F09xVUfSm7vwjNuf217f", - "XvohcvcSiQzNne5klOYR", - "qggTaBquDP6AWbDJ0t0c", - "VENEl2PjKFcfJd026VJI", - "4QfGvAM551kTHKbFZHG6", - "43bEkIQa3c6PoTIzuFIs", - "PQEa3eVF9OpxaEBlGjpT", - "Bf911TmARyP6oDWrqATg", - "SMOjqKYmxHXYvedCA7kc", - "2O9k7XNkHGIgQ48Ml5Q9", - "j5p6toahTPxKCIkkMVp0", - "9pG3k3HhdksD024aBIUJ", - "zZBCPOzCq6RGHNsBUfcY", - "pXFhiqulve6oZKFjzH0b", - "zHo18RaFDSxOo2mUA3lG", - "e3l8vbkjVogEjKhEOz8D", - "I7dSYYlGbJDKWpX2eYCE", - "nSyxFZsdsi6Qv3alJj3N", - "OKW0RgFDWxrPsuOFBX8s", - "UtAiuow38wWlfYVOVexE", - "mXHdVVhXV0H6LfbvxaId", - "06VyYns8lQwkCrRxXbHe", - "ZRYCVmYYhpzDnzH2WMLM", - "nyiGWFlZK0RCZxZhVG8o", - "m5Rzwx63dFaV2ChBb8wq", - "bW8ZXsxJ5SuLTC4g9bfF", - "OWVniAxWgUg2pFDrqDRc", - "ouZXRPAfLNv8Q9GD7MES", - "vqwRSUPUhRrPJizL6hCo", - "JBqIucQoFiZ2VLkz9idU", - "NKHAXcqzUbmCcL1pSZQC", - "SqNd2weyL7v8kubJi2xQ", - "KlmAAUtq6gDYXj5TGf5L", - "oLgU5vNer0WPfK8ACu4g", - "80jPGox77ZnnQsV9BqqE", - "d6Uyy5t6bnw6wtws8YiD", - "V09NW9x6dztjE8ae6bsg", - "oRkkc0eWjC5BAhPQXEJr", - "puo6qpm8IG6Rw3D1mary", - "U72msLgAbThkScWhPsN2", - "Wxih33pAxf32hXfVD8C4", - "gp5zjw4G70PZDx2uaR6c", - "JETBLaYh0RLNe48PA5o3", - "iYeOtg57924K801x0AiB", - "1yQbiNWYhjrqenOMzVoO", - "Z5qJnace6rp3epQAyOXH", - "Cdck3TzyEWP5oxT3S99B", - "J2BI6QhRm4EgrdrFH4cT", - "OLWaeUN72h8pe3cRtTSS", - "OUb7kQJBsDFKzDpEdztG", - "AbhDBDCWnswf3cdklfYQ", - "LUkSsOHKcheM2scouHce", - "vB9P1W4rkFiJbth6Nsat", - "IDzPbQtxYQ8vjLzgNiny", - "vFh3OxnldRk74QEWXZLa", - "Xr8n06UbMCT3AC8gZ5Ya", - "PUcUFAXjFC36Ma6e0EoB", - "I8P9oDqa2GEIoxLZMFpn", - "z4h0KXSI2nR0mDuStwpo", - "kfxY37hayV5z4GpzQfnZ" + "RwovlXQ1snDlQPmLcMtX", + "m1QKhB35vjvrw8cWDPIL", + "tu2LJwmqDSRmVKETE7P4", + "ZTppQoULmN7DmAoAGbkX", + "Ljet3MVkeWZEFT4cn8kZ", + "1gjEY4upDUqKEzTteSf8", + "nwVhyW0W8RoqIgeZHWII", + "O1SfaHkMKiAeKlCB3i3X", + "JCZLQOSbHloHSJCG1V4M", + "499eumzmxhxaGkr8UjiV", + "xt1QYhiDPIJvSKDAu9Bm", + "4Q244beSDOPrAR969i18", + "JyRV4qyujIpDGOFP23nk", + "ogGCUNFLKxEZL8Kj2hkR", + "TP2qyxIJO9yUlN2gkI2R", + "3njVrHwWoSz2anaSL91W", + "ekmvmUi7KzLoIKNw4QD6", + "nCP76KQeCofFa4OSxRpM", + "CzPW5YkbnLoY58WLbGXM", + "UxI2wSnBcg4mVTNjZ9Y5", + "rzOfJ52EEWKuPjgvmRJi", + "oGlxr9cwD49ExsXh604B", + "J9EbzvRUcnZwAoSxJhq2", + "tFVAoBwHoHYqmxHoHbWT", + "k2VVuS7T3INCBP6J4Hjo", + "OhDxPFqSNtXxMM9yJdAv", + "qjKIJ5fRfruj2iBua5qd", + "hcqeXnjuq0Qq4E8xn8Ib", + "mGVIPXwXQ444vvLBGvna", + "MYR4uz7mvdVfLaARDN2T", + "kdyc0yE3UdjJ3biajnJj", + "LshtBGUiN6b4X6dVjVrj", + "fCVDq2WvdynfFGp4R0E4", + "ossClcGwq5wLRUF8QN1r", + "9TDxe9FjLo1fAqOMGv0k", + "yK7vrtg9mbtYYE6K9qwa", + "osU8sYsgRa6IKO4xevDw", + "EVtiG3yOFeC32NLKSbcB", + "102WCDM79qgEfC3rdvrZ", + "zKaInP73dDDL3Sj0RnU3", + "3JQp2vezxP5JEA5WaCsG", + "Gjws7sVd16gJQWIbCSE9", + "bW6iZn9iQZPfv4sjHpiy", + "e1FF6FNJ1TVkTK40kK0Y", + "loVi3ZlegdXKt5KuIWD9", + "gbf0a1pg33NBhVfeA9Jd", + "ZxklYSrneuEKWmGr5q0H", + "8pY9zKWRSXYztTtX2Dsu", + "uDTnyMyuZ8dTKXtZfwnD", + "JpLP8mH5d9Bjl8nT7Eot", + "H8sWEwjn5cNP336XRjBm", + "FgGZRHNjiNFSjA9jSNpd", + "JoWsHlySX30ytc4Nrjgp", + "z9EUPGqJJ2psxI4w5eDx", + "THAfYCTS7kdoOwdMbem9", + "tW2dIuFa6xmKk23HW0bB", + "PhZ9g08JZPF4wJkujckI", + "16HEc1MMs08jOHElnffy", + "QhlWwcgkIcgeXWOFLhhE", + "GFmxh1hjvUW7PWimDZ2f", + "BYEfxiWXia1NNjLhUc8q", + "GC3qOEl7VjK7DvQc5J2L", + "Bn9f8P5jRslBtmKiZPzy", + "wPRahJuThxpluEIx81lU", + "9SWXp9YbDFEBlcuVZUaC", + "yLilVeNK1SCG3ASFVbmq", + "FT276Es24XGJhUvk5qRx", + "sjp8ncb9dLkdoL8LLZhv", + "eDkmN2xZ39mHoTNaKNxE", + "8GAe0ewJl0zuYZSjEEXR", + "dYux7hQAPNoptIU2YDW6", + "OF8pMK4BwiSTAwsZdSaz", + "IGJA9flaWsVkVjB3goID", + "RZlMG1eR6ZjTNncj5Oxu", + "7f5RIlXE0l1PWC8SFjet", + "phqN9Aj9RWiJJKVBGoac", + "8CZDNMWZNLavWYudTFzU", + "6vnLzbizEMz6ONGGIXsh", + "yyHYfMpGlDP4jGYHwPJD", + "cZ0C7GKi6fxGLI23aBcp", + "7jspiPcDFpcTI5eoXOa4", + "T54xCtrhiuSutpyoyzm9", + "W9Vo7fweW7TxBeNi0Wkp", + "CJGkWtCwUQ3VRlt0Fozl", + "H7UrsRkQO65anpwDSHhL", + "uAHUDeX0dQH4d5jEBd3t", + "EagFaADXRsIZ1PJk3D2a", + "N2Uk836xTQ34jSyi7WNE", + "3hV3IeIFQoVXy941lzqm", + "ROc9wJQgwY29jQb9MQJs", + "EPi6OLT3hfhLmVRgUChO", + "K4U5dKsvrofd7mY3l5lp", + "I2MNEbvABg7tZG2tqtx2", + "qnxDgpgNrK4mid4QoJLz", + "pTT23LxARdAXVuW0qqCx", + "EkVVOXSSX0yvgcnngYZ6", + "lQShCmtrpJwb8rOtKGMQ", + "pdq1TKljIVxfcakWtpdg", + "KLM0PRS7rMp1SQUeK8c6", + "Ho4uzFCzMwrg7t2Pmoxz", + "xZQDDgfO9HqqAd03nA90", + "kXpETSkGLtGzJhaQ946K", + "xcJwuXgmCoawcGuwJLaE", + "UbO651ujKWbhulALO5LC", + "cmGfI01YjgMgPPyxwajS", + "vhekh5AvXYXt2NeDqJPh", + "jhKi9ZwU0VubFCiGDd2J", + "y2DWsfpdK3FZ8N4FkTAW", + "JYj3T6aQjkYTdGirBWr3", + "cGI2HUgXQwzMMaDhEA6D", + "XD9UuN053Zj5EGgi6ocm", + "M64qxq184jKHqk8vlW8J", + "oo3DkGGKkx57w7ypr8pp", + "UVHNcXJr93Lyo2pjD94F", + "k8GMtUxynpstFQ4FNvWi", + "OyCCPtZ17sYgc9FYLJrS", + "Yz0wD1llAL3a33txe7lX", + "Td0WEyqKEelw5E7wajRD", + "twl0YyWCZ1tDe3P8YvYx", + "js3FbR48v08Lc6FZD29D", + "dqDv6kLFdvvcE4Y5s2oz", + "nafPV1EopdZRibEjxkMU", + "VutMAZoKihv9pu5DNYXe", + "Ee7OFTqUxbapi7sBKKQ5", + "C1wIwOmbwjqTGoNiOEuL", + "EyNbtpcUTQBAHPoaVku1", + "c78cAxCXU8gwbvwcpC7F", + "QxwVWcjHD5hAlcXYw2UU", + "zz3PhVgI0dWvmKEm5VaD", + "GrrpSofZ5n3ZutyPJfo7", + "6DGsMGG2vnupr6zgaAPo", + "QwCXcdq2utJLdqDg8Bt3", + "8SFeow3rt7shOQGkwG9Y", + "lXmRfrosiEGKyqbsyG15", + "jsSoutG1g5hzrzjR2U2s", + "Jnau7OBpvvY6VyQRQnaY", + "RDWZwwFQSGKB5sQ9bGI3", + "C2qFceAcoks7Oqq8RDT1", + "4hKjWW1NKEqWoVAB5g9Q", + "Ia5sNUqzWcUEfQszYVCT", + "F09xVUfSm7vwjNuf217f", + "XvohcvcSiQzNne5klOYR", + "qggTaBquDP6AWbDJ0t0c", + "VENEl2PjKFcfJd026VJI", + "4QfGvAM551kTHKbFZHG6", + "43bEkIQa3c6PoTIzuFIs", + "PQEa3eVF9OpxaEBlGjpT", + "Bf911TmARyP6oDWrqATg", + "SMOjqKYmxHXYvedCA7kc", + "2O9k7XNkHGIgQ48Ml5Q9", + "j5p6toahTPxKCIkkMVp0", + "9pG3k3HhdksD024aBIUJ", + "zZBCPOzCq6RGHNsBUfcY", + "pXFhiqulve6oZKFjzH0b", + "zHo18RaFDSxOo2mUA3lG", + "e3l8vbkjVogEjKhEOz8D", + "I7dSYYlGbJDKWpX2eYCE", + "nSyxFZsdsi6Qv3alJj3N", + "OKW0RgFDWxrPsuOFBX8s", + "UtAiuow38wWlfYVOVexE", + "mXHdVVhXV0H6LfbvxaId", + "06VyYns8lQwkCrRxXbHe", + "ZRYCVmYYhpzDnzH2WMLM", + "nyiGWFlZK0RCZxZhVG8o", + "m5Rzwx63dFaV2ChBb8wq", + "bW8ZXsxJ5SuLTC4g9bfF", + "OWVniAxWgUg2pFDrqDRc", + "ouZXRPAfLNv8Q9GD7MES", + "vqwRSUPUhRrPJizL6hCo", + "JBqIucQoFiZ2VLkz9idU", + "NKHAXcqzUbmCcL1pSZQC", + "SqNd2weyL7v8kubJi2xQ", + "KlmAAUtq6gDYXj5TGf5L", + "oLgU5vNer0WPfK8ACu4g", + "80jPGox77ZnnQsV9BqqE", + "d6Uyy5t6bnw6wtws8YiD", + "V09NW9x6dztjE8ae6bsg", + "oRkkc0eWjC5BAhPQXEJr", + "puo6qpm8IG6Rw3D1mary", + "U72msLgAbThkScWhPsN2", + "Wxih33pAxf32hXfVD8C4", + "gp5zjw4G70PZDx2uaR6c", + "JETBLaYh0RLNe48PA5o3", + "iYeOtg57924K801x0AiB", + "1yQbiNWYhjrqenOMzVoO", + "Z5qJnace6rp3epQAyOXH", + "Cdck3TzyEWP5oxT3S99B", + "J2BI6QhRm4EgrdrFH4cT", + "OLWaeUN72h8pe3cRtTSS", + "OUb7kQJBsDFKzDpEdztG", + "AbhDBDCWnswf3cdklfYQ", + "LUkSsOHKcheM2scouHce", + "vB9P1W4rkFiJbth6Nsat", + "IDzPbQtxYQ8vjLzgNiny", + "vFh3OxnldRk74QEWXZLa", + "Xr8n06UbMCT3AC8gZ5Ya", + "PUcUFAXjFC36Ma6e0EoB", + "I8P9oDqa2GEIoxLZMFpn", + "z4h0KXSI2nR0mDuStwpo", + "kfxY37hayV5z4GpzQfnZ" ); @Test @Order(3) public void checkExactHandlesManyValues() { - ConceptsProcessor.ExactFilterValueResult actual = searchProcessor.findExact(FILTER, MANY_VALUES.stream().toList()); + ConceptsProcessor.ExactFilterValueResult actual = searchProcessor.findExact( + FILTER, + MANY_VALUES.stream().toList()); - assertThat(actual).usingRecursiveComparison().isEqualTo(new ConceptsProcessor.ExactFilterValueResult(List.of(),MANY_VALUES)); + assertThat(actual).usingRecursiveComparison() + .isEqualTo( + new ConceptsProcessor.ExactFilterValueResult(List.of(), MANY_VALUES)); } @Test @Order(3) public void checkExactFailsValueTooLong() { - assertThatThrownBy( () -> searchProcessor.findExact(FILTER, List.of(String.join("",MANY_VALUES)))).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> searchProcessor.findExact(FILTER, List.of(String.join("", MANY_VALUES)))).isInstanceOf( + IllegalArgumentException.class); } @Test diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/ConfigOverride.java b/backend/src/test/java/com/bakdata/conquery/util/support/ConfigOverride.java index e8f93157a66..be8d07c0286 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/ConfigOverride.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/ConfigOverride.java @@ -38,8 +38,8 @@ public static void configureRandomPorts(ConqueryConfig config) { // set random open ports final Collection connectorFactories = CollectionUtils.union( - ((DefaultServerFactory) config.getServerFactory()).getAdminConnectors(), - ((DefaultServerFactory) config.getServerFactory()).getApplicationConnectors() + ((DefaultServerFactory) config.getServerFactory()).getAdminConnectors(), + ((DefaultServerFactory) config.getServerFactory()).getApplicationConnectors() ); for (ConnectorFactory con : connectorFactories) { diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/ConqueryAuthenticationFilter.java b/backend/src/test/java/com/bakdata/conquery/util/support/ConqueryAuthenticationFilter.java index aef66c7c2a4..f28ce4b9c56 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/ConqueryAuthenticationFilter.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/ConqueryAuthenticationFilter.java @@ -1,9 +1,9 @@ package com.bakdata.conquery.util.support; -import java.util.function.Supplier; import jakarta.ws.rs.client.ClientRequestContext; import jakarta.ws.rs.client.ClientRequestFilter; import jakarta.ws.rs.core.HttpHeaders; +import java.util.function.Supplier; /** * Simple filter for http client in test to provide authentication information. diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java b/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java index 690b997ee94..ac75be2cf57 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java @@ -1,10 +1,10 @@ package com.bakdata.conquery.util.support; +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.core.UriBuilder; import java.io.File; import java.util.List; import java.util.Map; -import jakarta.ws.rs.client.Client; -import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.commands.PreprocessorCommand; import com.bakdata.conquery.integration.json.TestDataImporter; @@ -42,24 +42,32 @@ public class StandaloneSupport implements TestSupport { public void preprocessTmp(File tmpDir, List descriptions) throws Exception { final Environment env = testConquery.getDropwizard().getEnvironment(); final net.sourceforge.argparse4j.inf.Namespace namespace = new net.sourceforge.argparse4j.inf.Namespace( - Map.of( - "in", tmpDir, - "out", tmpDir, - "desc", descriptions, - "buckets", 10, - "strict", true, - "fast-fail", true - ) + Map.of( + "in", + tmpDir, + "out", + tmpDir, + "desc", + descriptions, + "buckets", + 10, + "strict", + true, + "fast-fail", + true + ) ); // We use this to change the visibility of the run method, hence it cannot be instantiated. - new PreprocessorCommand(MoreExecutors.newDirectExecutorService()){ + new PreprocessorCommand(MoreExecutors.newDirectExecutorService()) { @Override - public void run(Environment environment, net.sourceforge.argparse4j.inf.Namespace namespace, ConqueryConfig config) throws Exception { + public void run( + Environment environment, + net.sourceforge.argparse4j.inf.Namespace namespace, + ConqueryConfig config) throws Exception { super.run(environment, namespace, config); } - } - .run(env, namespace, config); + }.run(env, namespace, config); } public NamespaceStorage getNamespaceStorage() { @@ -81,10 +89,7 @@ public Client getClient() { } public UriBuilder defaultApiURIBuilder() { - return UriBuilder.fromPath("api") - .host("localhost") - .scheme("http") - .port(getLocalPort()); + return UriBuilder.fromPath("api").host("localhost").scheme("http").port(getLocalPort()); } /** @@ -97,10 +102,7 @@ public int getLocalPort() { } public UriBuilder defaultAdminURIBuilder() { - return UriBuilder.fromPath("admin") - .host("localhost") - .scheme("http") - .port(getAdminPort()); + return UriBuilder.fromPath("admin").host("localhost").scheme("http").port(getAdminPort()); } /** @@ -112,7 +114,9 @@ public int getAdminPort() { return testConquery.getDropwizard().getAdminPort(); } - public enum Mode {WORKER, SQL} + public enum Mode { + WORKER, SQL + } } diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/TestBootstrappingConquery.java b/backend/src/test/java/com/bakdata/conquery/util/support/TestBootstrappingConquery.java index 7c37dd00455..f0395398af9 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/TestBootstrappingConquery.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/TestBootstrappingConquery.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.util.support; +import jakarta.validation.Validator; import java.io.File; import java.io.IOException; import java.time.Clock; @@ -16,7 +17,6 @@ import io.dropwizard.core.setup.Bootstrap; import io.dropwizard.logging.common.BootstrapLogging; import io.dropwizard.logging.common.DropwizardLayout; -import jakarta.validation.Validator; public class TestBootstrappingConquery extends Conquery { @@ -30,15 +30,21 @@ public static Bootstrap createTestBootstrapConquery(File tmpDir) bootstrap.setConfigurationFactoryFactory(new DefaultConfigurationFactoryFactory<>() { @Override - public ConfigurationFactory create(Class klass, Validator validator, ObjectMapper objectMapper, String propertyPrefix) { + public ConfigurationFactory create( + Class klass, + Validator validator, + ObjectMapper objectMapper, + String propertyPrefix) { return new YamlConfigurationFactory( - klass, - validator, - configureObjectMapper(objectMapper.copy()), - propertyPrefix + klass, + validator, + configureObjectMapper(objectMapper.copy()), + propertyPrefix ) { @Override - protected ConqueryConfig build(JsonNode node, String path) throws IOException, ConfigurationException { + protected ConqueryConfig build( + JsonNode node, + String path) throws IOException, ConfigurationException { final ConqueryConfig config = super.build(node, path); ConfigOverride.configurePathsAndLogging(config, tmpDir); return config; diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java index 8abbc29a5e0..4b5dd370d5f 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java @@ -4,6 +4,9 @@ import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.fail; +import jakarta.validation.Validator; +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.core.UriBuilder; import java.io.File; import java.util.HashSet; import java.util.List; @@ -13,9 +16,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Supplier; -import jakarta.validation.Validator; -import jakarta.ws.rs.client.Client; -import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.commands.DistributedStandaloneCommand; import com.bakdata.conquery.commands.ShardNode; @@ -39,7 +39,6 @@ import com.bakdata.conquery.resources.admin.rest.AdminDatasetProcessor; import com.bakdata.conquery.resources.admin.rest.AdminProcessor; import com.bakdata.conquery.util.io.Cloner; -import com.google.common.base.Stopwatch; import io.dropwizard.client.JerseyClientBuilder; import io.dropwizard.core.cli.Command; import io.dropwizard.testing.DropwizardTestSupport; @@ -78,24 +77,20 @@ public class TestConquery { public static void waitUntil(Supplier condition) { final AtomicLong done = new AtomicLong(); - await().atMost(10, TimeUnit.SECONDS) - .pollInterval(2, TimeUnit.MILLISECONDS) - .until(() -> { - if (!condition.get()) { - return false; - } - else { - return done.getAndIncrement() > 5; - } - }); + await().atMost(10, TimeUnit.SECONDS).pollInterval(2, TimeUnit.MILLISECONDS).until(() -> { + if (!condition.get()) { + return false; + } else { + return done.getAndIncrement() > 5; + } + }); } public synchronized StandaloneSupport openDataset(DatasetId datasetId) { try { log.info("loading dataset"); return createSupport(datasetId, datasetId.getName()); - } - catch (Exception e) { + } catch (Exception e) { return fail("Failed to open dataset " + datasetId, e); } } @@ -120,24 +115,24 @@ private StandaloneSupport buildSupport(DatasetId datasetId, String name, Standal if (!localTmpDir.mkdir()) { throw new IllegalStateException("Could not create directory for Support:" + localTmpDir); } - } - else { + } else { log.info("Reusing existing folder {} for Support", localTmpDir.getPath()); } - ConqueryConfig - localCfg = - Cloner.clone(config, Map.of(Validator.class, standaloneCommand.getManagerNode().getEnvironment().getValidator()), IntegrationTests.MAPPER); + ConqueryConfig localCfg = Cloner.clone( + config, + Map.of(Validator.class, standaloneCommand.getManagerNode().getEnvironment().getValidator()), + IntegrationTests.MAPPER); StandaloneSupport support = new StandaloneSupport( - mode, - this, - ns, - ns.getStorage().getDataset().getId(), - localTmpDir, - localCfg, - // Getting the User from AuthorizationConfig - testDataImporter + mode, + this, + ns, + ns.getStorage().getDataset().getId(), + localTmpDir, + localCfg, + // Getting the User from AuthorizationConfig + testDataImporter ); openSupports.add(support); @@ -150,7 +145,12 @@ private synchronized StandaloneSupport buildDistributedSupport(DatasetId dataset ClusterState clusterState = manager.getConnectionManager().getClusterState(); assertThat(clusterState.getShardNodes()).hasSize(2); - waitUntil(() -> clusterState.getWorkerHandlers().get(datasetId).getWorkers().size() == clusterState.getShardNodes().size()); + waitUntil( + () -> clusterState.getWorkerHandlers() + .get( + datasetId) + .getWorkers() + .size() == clusterState.getShardNodes().size()); return buildSupport(datasetId, name, StandaloneSupport.Mode.WORKER); } @@ -170,8 +170,7 @@ public synchronized StandaloneSupport getSupport(String name) { DatasetId datasetId = getDatasetRegistry().get(new DatasetId(name)).getDataset().getId(); return createSupport(datasetId, name); - } - catch (Exception e) { + } catch (Exception e) { return fail("Failed to create a support for " + name, e); } } @@ -181,10 +180,7 @@ public DatasetRegistry getDatasetRegistry() { } public UriBuilder defaultAdminURIBuilder() { - return UriBuilder.fromPath("admin") - .host("localhost") - .scheme("http") - .port(dropwizard.getAdminPort()); + return UriBuilder.fromPath("admin").host("localhost").scheme("http").port(dropwizard.getAdminPort()); } @SneakyThrows @@ -197,9 +193,13 @@ private boolean isBusy() { boolean busy; busy = standaloneCommand.getManagerNode().getJobManager().isSlowWorkerBusy(); - busy |= standaloneCommand.getManager().getDatasetRegistry().getNamespaces().stream() - .map(Namespace::getExecutionManager) - .anyMatch(ExecutionManager::hasRunningQueries); + busy |= standaloneCommand.getManager() + .getDatasetRegistry() + .getNamespaces() + .stream() + .map( + Namespace::getExecutionManager) + .anyMatch(ExecutionManager::hasRunningQueries); for (Namespace namespace : standaloneCommand.getManagerNode().getDatasetRegistry().getNamespaces()) { busy |= namespace.getJobManager().isSlowWorkerBusy(); @@ -229,8 +229,7 @@ public void beforeAll() throws Exception { dropwizard = new DropwizardTestSupport<>(TestBootstrappingConquery.class, config, app -> { if (config.getSqlConnectorConfig().isEnabled()) { standaloneCommand = new SqlStandaloneCommand(); - } - else { + } else { standaloneCommand = new DistributedStandaloneCommand(); } standaloneCommand.setClock(MockClock.get()); @@ -241,16 +240,17 @@ public void beforeAll() throws Exception { dropwizard.before(); // create HTTP client for api tests - client = new JerseyClientBuilder(getDropwizard().getEnvironment()) - .withProperty(ClientProperties.CONNECT_TIMEOUT, 10000) - .withProperty(ClientProperties.READ_TIMEOUT, 10000) - .build("test client"); + client = new JerseyClientBuilder(getDropwizard().getEnvironment()).withProperty( + ClientProperties.CONNECT_TIMEOUT, + 10000).withProperty(ClientProperties.READ_TIMEOUT, 10000).build("test client"); // The test user is recreated after each test, in the storage, but its id stays the same. // Here we register the client filter once for that test user id. UserId testUserId = config.getAuthorizationRealms().getInitialUsers().getFirst().createId(); - client.register(new ConqueryAuthenticationFilter(() -> getAuthorizationController().getConqueryTokenRealm().createTokenForUser(testUserId))); + client.register( + new ConqueryAuthenticationFilter( + () -> getAuthorizationController().getConqueryTokenRealm().createTokenForUser(testUserId))); testUser = getMetaStorage().getUser(testUserId); } @@ -299,7 +299,13 @@ public void beforeEach() { // MetaStorage is cleared after each test, so we need to add the test user again final MetaStorage storage = standaloneCommand.getManagerNode().getMetaStorage(); - testUser = standaloneCommand.getManagerNode().getConfig().getAuthorizationRealms().getInitialUsers().getFirst().createOrOverwriteUser(storage); + testUser = standaloneCommand.getManagerNode() + .getConfig() + .getAuthorizationRealms() + .getInitialUsers() + .getFirst() + .createOrOverwriteUser( + storage); } public Validator getValidator() { @@ -319,9 +325,6 @@ public AdminDatasetProcessor getAdminDatasetsProcessor() { } public UriBuilder defaultApiURIBuilder() { - return UriBuilder.fromPath("api") - .host("localhost") - .scheme("http") - .port(dropwizard.getLocalPort()); + return UriBuilder.fromPath("api").host("localhost").scheme("http").port(dropwizard.getLocalPort()); } } diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/TestLoggingFactory.java b/backend/src/test/java/com/bakdata/conquery/util/support/TestLoggingFactory.java index 094c527c93a..cd0a407cef3 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/TestLoggingFactory.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/TestLoggingFactory.java @@ -61,13 +61,18 @@ public void configure(MetricRegistry metricRegistry, String name) { final ConsoleAppenderFactory consoleAppender = new ConsoleAppenderFactory<>(); consoleAppender.setLogFormat(LOG_PATTERN); - root.addAppender(consoleAppender.build(loggerContext, name, new DropwizardLayoutFactory(), new ThresholdLevelFilterFactory(), new AsyncLoggingEventAppenderFactory())); + root.addAppender( + consoleAppender.build( + loggerContext, + name, + new DropwizardLayoutFactory(), + new ThresholdLevelFilterFactory(), + new AsyncLoggingEventAppenderFactory())); StatusPrinter.setPrintStream(configurationErrorsStream); try { StatusPrinter.printIfErrorsOccured(loggerContext); - } - finally { + } finally { StatusPrinter.setPrintStream(System.out); } @@ -127,8 +132,7 @@ private void flushAppender(Appender appender) { // appender.stop() if the appender isn't able to flush. asyncAppender.addWarn(asyncAppender.getNumberOfElementsInQueue() + " events may be discarded"); } - } - catch (InterruptedException ignored) { + } catch (InterruptedException ignored) { // If the thread waiting for the logs to be flushed is aborted then // user clearly wants the application to quit now, so stop trying // to flush any appenders diff --git a/backend/src/test/java/com/bakdata/conquery/validation/CaffeineValidationTest.java b/backend/src/test/java/com/bakdata/conquery/validation/CaffeineValidationTest.java index 1e20162e495..41f88d50db0 100644 --- a/backend/src/test/java/com/bakdata/conquery/validation/CaffeineValidationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/validation/CaffeineValidationTest.java @@ -2,10 +2,10 @@ import static org.assertj.core.api.Assertions.assertThat; -import java.util.Set; import jakarta.validation.ConstraintViolation; import jakarta.validation.Validation; import jakarta.validation.Validator; +import java.util.Set; import com.bakdata.conquery.util.validation.ValidCaffeineSpec; import org.junit.jupiter.api.Test; @@ -24,7 +24,8 @@ void correctSpec() { @Test void correctSoftSpec() { - Set> softValues = validator.validate(new ContainerSoft("softValues,maximumSize=1")); + Set> softValues = validator.validate( + new ContainerSoft("softValues,maximumSize=1")); assertThat(softValues).hasSize(0); @@ -39,7 +40,8 @@ void missingSoftValuesSpec() { @Test void unparsableSpec() { - Set> softValues = validator.validate(new ContainerSoft("maximumSize=1=2,softValues")); + Set> softValues = validator.validate( + new ContainerSoft("maximumSize=1=2,softValues")); assertThat(softValues).hasSize(2); } @@ -52,10 +54,11 @@ void unparsableAndMissingSoftValuesSpec() { assertThat(softValues).hasSize(3); } - record ContainerSoft(@ValidCaffeineSpec(softValue = true) String spec) {} - - record Container(@ValidCaffeineSpec() String spec) {} + record ContainerSoft(@ValidCaffeineSpec(softValue = true) String spec) { + } + record Container(@ValidCaffeineSpec() String spec) { + } } diff --git a/conquery_eclipse_format_setting.xml b/conquery_eclipse_format_setting.xml new file mode 100644 index 00000000000..e8544eb86d6 --- /dev/null +++ b/conquery_eclipse_format_setting.xml @@ -0,0 +1,381 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pom.xml b/pom.xml index 860dda8e2fe..9f8eb3eb79e 100644 --- a/pom.xml +++ b/pom.xml @@ -23,6 +23,8 @@ ${java.required} ${java.required} 3.8.1 + 3.9.0 + 4.40 yyyy-MM-dd'T'HH:mm:ssXXX ${maven.build.timestamp} 0.0.0-SNAPSHOT @@ -56,6 +58,35 @@ + + com.diffplug.spotless + spotless-maven-plugin + ${spotless-maven-plugin.version} + + + UNIX + + ${eclipse-jdt-formatter.version} + ${maven.multiModuleProjectDirectory}/conquery_eclipse_format_setting.xml + + + + \#jakarta|\#java|\#javax|\#,jakarta|java|javax, + + + + + + + + + check-java-formatting + + check + + + + org.apache.maven.plugins maven-compiler-plugin @@ -129,4 +160,4 @@ - \ No newline at end of file + From 6b92ed1b637b9f5a70ac68d3206dd7b583fb3290 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 27 Aug 2026 11:34:27 +0200 Subject: [PATCH 2/3] remove unused classes --- .../PostgreSqlIntervalPacker.java | 59 ------------------- .../com/bakdata/conquery/util/BufferUtil.java | 14 ----- .../io/jackson/SerializationBlockTest.java | 22 ------- 3 files changed, 95 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/util/BufferUtil.java delete mode 100644 backend/src/test/java/com/bakdata/conquery/io/jackson/SerializationBlockTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java deleted file mode 100644 index 8fae49bdb86..00000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java +++ /dev/null @@ -1,59 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.intervalpacking; - -import java.util.Optional; - -import com.bakdata.conquery.sql.conversion.dialect.IntervalPacker; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; -import lombok.RequiredArgsConstructor; - -/** - * PostgreSql supports interval packing with a native range function. - *

- * See && range function operator - */ -@RequiredArgsConstructor -public class PostgreSqlIntervalPacker implements IntervalPacker { - - private final SqlFunctionProvider functionProvider; - - @Override - public QueryStep aggregateAsValidityDate(IntervalPackingContext context) { - return aggregateDate(context, AggregationMode.VALIDITY_DATE); - } - - @Override - public QueryStep aggregateAsArbitrarySelect(IntervalPackingContext context) { - return aggregateDate(context, AggregationMode.ARBITRARY_SELECT); - } - - private QueryStep aggregateDate(IntervalPackingContext context, AggregationMode aggregationMode) { - - String sourceTableName = context.getTables().getPredecessor(IntervalPackingCteStep.INTERVAL_COMPLETE); - SqlIdColumns ids = context.getIds().qualify(sourceTableName); - ColumnDateRange qualifiedDaterange = context.getDaterange().qualify(sourceTableName); - ColumnDateRange aggregatedDaterange = this.functionProvider.aggregated(qualifiedDaterange); - - Selects.SelectsBuilder selectsBuilder = Selects.builder() - .ids(ids) - .sqlSelects(context.getCarryThroughSelects()); - - switch (aggregationMode) { - case VALIDITY_DATE -> selectsBuilder.validityDate(Optional.of(aggregatedDaterange)); - case ARBITRARY_SELECT -> selectsBuilder.sqlSelect(aggregatedDaterange); - } - - return QueryStep.builder() - .cteName(context.getTables().cteName(IntervalPackingCteStep.INTERVAL_COMPLETE)) - .selects(selectsBuilder.build()) - .fromTable(QueryStep.toTableLike(sourceTableName)) - .groupBy(ids.toFields()) - .predecessors(Optional.ofNullable(context.getPredecessor()).stream().toList()) - .build(); - } - - -} diff --git a/backend/src/main/java/com/bakdata/conquery/util/BufferUtil.java b/backend/src/main/java/com/bakdata/conquery/util/BufferUtil.java deleted file mode 100644 index 1cc4cc7f14e..00000000000 --- a/backend/src/main/java/com/bakdata/conquery/util/BufferUtil.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.bakdata.conquery.util; - -import java.util.Arrays; - -import lombok.experimental.UtilityClass; -import org.apache.mina.core.buffer.IoBuffer; - -@UtilityClass -public class BufferUtil { - - public static byte[] toBytes(IoBuffer buffer) { - return Arrays.copyOfRange(buffer.array(), buffer.arrayOffset()+ buffer.position(), buffer.arrayOffset()+buffer.limit()); - } -} diff --git a/backend/src/test/java/com/bakdata/conquery/io/jackson/SerializationBlockTest.java b/backend/src/test/java/com/bakdata/conquery/io/jackson/SerializationBlockTest.java deleted file mode 100644 index e1cf160e5b3..00000000000 --- a/backend/src/test/java/com/bakdata/conquery/io/jackson/SerializationBlockTest.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.bakdata.conquery.io.jackson; - -import org.junit.jupiter.api.Test; - -import lombok.Data; - -public class SerializationBlockTest { - @Test - public void testUnmarkedClass() { - //see #141 - /*assertThatThrownBy(() -> { - Jackson.MAPPER.writeValueAsString(new Unmarked()); - }) - .isInstanceOf(JsonProcessingException.class);*/ - } - - @Data - private static class Unmarked { - - private String test = "test"; - } -} From fb618d39cc09aa4ca0fc59a57a6abe88f77bcf86 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 27 Aug 2026 11:54:34 +0200 Subject: [PATCH 3/3] spotless --- .../bakdata/conquery/resources/api/ConceptsProcessor.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java index 206ec03d7dd..14383050261 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java @@ -108,7 +108,7 @@ public FrontendPreviewConfig getEntityPreviewFrontendConfig(DatasetId dataset) { final Namespace namespace = namespaces.get(dataset); final PreviewConfig previewConfig = namespace.getPreviewConfig(); - ConnectorId searchConnector = previewConfig.resolveSearchConnector(); + ConnectorId searchConnector = previewConfig.resolveSearchConnector(); // Connectors only act as bridge to table for the fronted, but also provide ConceptColumnT semantic return new FrontendPreviewConfig( @@ -128,7 +128,8 @@ public FrontendPreviewConfig getEntityPreviewFrontendConfig(DatasetId dataset) { id.resolve().getResolvedTable().getLabel())) .collect(Collectors.toSet()), previewConfig.getSearchFilters(), - config.getIdColumns().getIds().stream().map(ColumnConfig::getName).toList(),searchConnector.getConcept(), + config.getIdColumns().getIds().stream().map(ColumnConfig::getName).toList(), + searchConnector.getConcept(), searchConnector ); }