diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCExpiration.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCExpiration.java index 69db359efa6..172ffea4438 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCExpiration.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCExpiration.java @@ -84,10 +84,9 @@ public ActionType getActionType() { * - Days must be a positive number greater than zero if set * - Either days or date should be specified, but not both or neither * - The date value must conform to the ISO 8601 format - * - The date value must be in the future * - The date value must be at midnight UTC (00:00:00Z) * - * @param creationTime The creation time of the lifecycle configuration in milliseconds since epoch + * @param creationTime unused, kept for interface compatibility * @throws OMException if the validation fails */ @Override @@ -107,7 +106,7 @@ public void valid(long creationTime) throws OMException { daysInMilli = TimeUnit.DAYS.toMillis(days); } if (hasDate) { - validateExpirationDate(date, creationTime); + validateExpirationDate(date); } } @@ -115,24 +114,16 @@ public void valid(long creationTime) throws OMException { * Validates that the expiration date is: * - In the ISO 8601 format * - Includes both time and time zone (neither can be omitted) - * - In the future * - Represents midnight UTC (00:00:00Z) when converted to UTC. * * @param expirationDate The date string to validate - * @param creationTime The creation time to compare against in milliseconds since epoch * @throws OMException if the date is invalid */ - private void validateExpirationDate(String expirationDate, long creationTime) throws OMException { + private void validateExpirationDate(String expirationDate) throws OMException { try { ZonedDateTime parsedDate = ZonedDateTime.parse(expirationDate, DateTimeFormatter.ISO_DATE_TIME); // Convert to UTC for validation ZonedDateTime dateInUTC = parsedDate.withZoneSameInstant(ZoneOffset.UTC); - // The date value must conform to the ISO 8601 format, be in the future. - ZonedDateTime createDate = ZonedDateTime.ofInstant(Instant.ofEpochMilli(creationTime), ZoneOffset.UTC); - if (dateInUTC.isBefore(createDate)) { - throw new OMException("Invalid lifecycle configuration: 'Date' must be in the future " + createDate + "," + - dateInUTC, OMException.ResultCodes.INVALID_REQUEST); - } // Verify that the time is midnight UTC (00:00:00Z) if (!test && (dateInUTC.getHour() != 0 || diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java index ce9f56c30b8..a7c4c8763df 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java @@ -404,6 +404,10 @@ public OmLCFilter getFilter() { return filter; } + public List getActions() { + return actions; + } + public OmLCRule build() { return new OmLCRule(this); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleConfiguration.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleConfiguration.java index 60c181cdf95..eaca191e587 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleConfiguration.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleConfiguration.java @@ -294,6 +294,10 @@ public Builder setCreationTime(long creationTime) { return this; } + public List getRules() { + return rules; + } + public Builder addRule(OmLCRule rule) { this.rules.add(rule); return this; diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCExpiration.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCExpiration.java index 1aa3e2b5011..991fac7b4ad 100644 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCExpiration.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCExpiration.java @@ -123,22 +123,21 @@ public void testCreateInValidOmLCExpiration() { assertOMException(() -> exp7.build().valid(currentTime), INVALID_REQUEST, "'Date' must be in ISO 8601 format"); - // Testing for date in the past with creation time + // Past dates are valid per S3 spec (they immediately expire objects) OmLCExpiration.Builder exp8 = new OmLCExpiration.Builder() .setDate(getFutureDateString(-1)); - assertOMException(() -> exp8.build().valid(currentTime), INVALID_REQUEST, - "'Date' must be in the future"); + assertDoesNotThrow(() -> exp8.build().valid(currentTime)); OmLCExpiration.Builder exp9 = new OmLCExpiration.Builder() .setDays(0); assertOMException(() -> exp9.build().valid(currentTime), INVALID_REQUEST, "'Days' for Expiration action must be a positive integer"); - // 1 minute ago with creation time + // A date that is not midnight UTC is rejected regardless of being past or future OmLCExpiration.Builder exp10 = new OmLCExpiration.Builder() .setDate(getFutureDateString(0, 0, -1)); assertOMException(() -> exp10.build().valid(currentTime), INVALID_REQUEST, - "'Date' must be in the future"); + "'Date' must represent midnight UTC"); } @Test diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketCrudHandler.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketCrudHandler.java index 23c4da2cfa2..14c7870867f 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketCrudHandler.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketCrudHandler.java @@ -197,7 +197,9 @@ public Response putBucketLifecycleConfiguration(S3RequestContext context, String null, null, null, null, body); OmLifecycleConfiguration lcc = s3LifecycleConfiguration.toOmLifecycleConfiguration(ozoneBucket); - ozoneBucket.setLifecycleConfiguration(lcc); + if (lcc != null) { + ozoneBucket.setLifecycleConfiguration(lcc); + } } catch (WebApplicationException ex) { throw S3ErrorTable.newError(S3ErrorTable.MALFORMED_XML, bucketName); } catch (OMException ex) { diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3LifecycleConfiguration.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3LifecycleConfiguration.java index de1de9616a7..fa8501c681f 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3LifecycleConfiguration.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3LifecycleConfiguration.java @@ -79,6 +79,9 @@ public static class Rule { @XmlElement(name = "Filter") private Filter filter; + @XmlElement(name = "NoncurrentVersionExpiration") + private NoncurrentVersionExpiration noncurrentVersionExpiration; + public String getId() { return id; } @@ -126,6 +129,14 @@ public Filter getFilter() { public void setFilter(Filter filter) { this.filter = filter; } + + public NoncurrentVersionExpiration getNoncurrentVersionExpiration() { + return noncurrentVersionExpiration; + } + + public void setNoncurrentVersionExpiration(NoncurrentVersionExpiration noncurrentVersionExpiration) { + this.noncurrentVersionExpiration = noncurrentVersionExpiration; + } } /** @@ -140,6 +151,9 @@ public static class Expiration { @XmlElement(name = "Date") private String date; + @XmlElement(name = "ExpiredObjectDeleteMarker") + private Boolean expiredObjectDeleteMarker; + public Integer getDays() { return days; } @@ -155,6 +169,36 @@ public String getDate() { public void setDate(String date) { this.date = date; } + + public Boolean getExpiredObjectDeleteMarker() { + return expiredObjectDeleteMarker; + } + + public void setExpiredObjectDeleteMarker(Boolean expiredObjectDeleteMarker) { + this.expiredObjectDeleteMarker = expiredObjectDeleteMarker; + } + + public boolean isUnsupported() { + return Boolean.TRUE.equals(expiredObjectDeleteMarker) && days == null && date == null; + } + } + + /** + * NoncurrentVersionExpiration entity for lifecycle rule (accepted but not yet enforced). + */ + @XmlAccessorType(XmlAccessType.FIELD) + @XmlRootElement(name = "NoncurrentVersionExpiration") + public static class NoncurrentVersionExpiration { + @XmlElement(name = "NoncurrentDays") + private Integer noncurrentDays; + + public Integer getNoncurrentDays() { + return noncurrentDays; + } + + public void setNoncurrentDays(Integer noncurrentDays) { + this.noncurrentDays = noncurrentDays; + } } /** @@ -291,7 +335,15 @@ public OmLifecycleConfiguration toOmLifecycleConfiguration(OzoneBucket ozoneBuck .setBucket(ozoneBucket.getName()); for (Rule rule : getRules()) { - builder.addRule(convertToOmRule(rule)); + OmLCRule omRule = convertToOmRule(rule); + if (omRule != null) { + builder.addRule(omRule); + } + } + + if (builder.getRules().isEmpty()) { + // All rules used unsupported-but-valid S3 actions; accept the request without persisting. + return null; } return builder.build(); @@ -307,9 +359,11 @@ public OmLifecycleConfiguration toOmLifecycleConfiguration(OzoneBucket ozoneBuck /** * Converts a single S3 lifecycle rule to Ozone internal rule representation. + * Returns null when the rule contains only S3 elements not yet supported by Ozone + * (e.g. NoncurrentVersionExpiration, ExpiredObjectDeleteMarker). * * @param rule the S3 lifecycle rule - * @return OmLCRule internal rule representation + * @return OmLCRule internal rule representation, or null if the rule uses only unsupported actions */ private OmLCRule convertToOmRule(Rule rule) throws OMException, OS3Exception { if (rule.getStatus() == null || rule.getStatus().isEmpty()) { @@ -317,21 +371,34 @@ private OmLCRule convertToOmRule(Rule rule) throws OMException, OS3Exception { "The Status element is required in LifecycleConfiguration"); } + boolean hasUnsupportedAction = false; + OmLCRule.Builder builder = new OmLCRule.Builder() .setEnabled("Enabled".equals(rule.getStatus())) .setId(rule.getId()) .setPrefix(rule.getPrefix()); if (rule.getExpiration() != null) { - builder.addAction(convertToOmExpiration(rule.getExpiration())); + if (rule.getExpiration().isUnsupported()) { + hasUnsupportedAction = true; + } else { + builder.addAction(convertToOmExpiration(rule.getExpiration())); + } } if (rule.getAbortIncompleteMultipartUpload() != null) { builder.addAction(convertToOmAbortIncompleteMultipartUpload(rule.getAbortIncompleteMultipartUpload())); } + if (rule.getNoncurrentVersionExpiration() != null) { + hasUnsupportedAction = true; + } if (rule.getFilter() != null) { builder.setFilter(convertToOmFilter(rule.getFilter())); } + if (builder.getActions().isEmpty() && hasUnsupportedAction) { + return null; + } + return builder.build(); } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationPut.java index 818df7f5e97..1307c7d7edc 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationPut.java @@ -251,6 +251,62 @@ public void testPutLifecycleWithBothExpirationAndAbort() throws Exception { assertEquals(HTTP_OK, bucketEndpoint.put("bucket1", withBothExpirationAndAbort()).getStatus()); } + @Test + public void testPutLifecycleWithPastExpirationDate() throws Exception { + // S3 accepts past dates; they mean objects expired immediately + String xml = "" + + "rule1test1/Enabled" + + "2017-09-27T00:00:00+00:00" + + ""; + assertEquals(HTTP_OK, + bucketEndpoint.put("bucket1", new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))).getStatus()); + } + + @Test + public void testPutLifecycleWithExpiredObjectDeleteMarker() throws Exception { + // ExpiredObjectDeleteMarker is a valid S3 action; Ozone accepts but does not enforce it yet + String xml = "" + + "rule1test1/Enabled" + + "true" + + ""; + assertEquals(HTTP_OK, + bucketEndpoint.put("bucket1", new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))).getStatus()); + } + + @Test + public void testPutLifecycleWithExpiredObjectDeleteMarkerAndFilter() throws Exception { + String xml = "" + + "rule1fooEnabled" + + "true" + + ""; + assertEquals(HTTP_OK, + bucketEndpoint.put("bucket1", new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))).getStatus()); + } + + @Test + public void testPutLifecycleWithExpiredObjectDeleteMarkerAndEmptyFilter() throws Exception { + String xml = "" + + "rule1Enabled" + + "true" + + ""; + assertEquals(HTTP_OK, + bucketEndpoint.put("bucket1", new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))).getStatus()); + } + + @Test + public void testPutLifecycleWithNoncurrentVersionExpiration() throws Exception { + // NoncurrentVersionExpiration is a valid S3 action; Ozone accepts but does not enforce it yet + String xml = "" + + "rule1past/Enabled" + + "2" + + "" + + "rule2future/Enabled" + + "3" + + ""; + assertEquals(HTTP_OK, + bucketEndpoint.put("bucket1", new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))).getStatus()); + } + @Test public void testPutInvalidAbortIncompleteMultipartUploadConfig() throws Exception { // Test with zero days - should fail