Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import java.io.IOException;
import java.lang.reflect.Method;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
import java.net.URLEncoder;
import java.security.GeneralSecurityException;
Expand Down Expand Up @@ -284,7 +285,7 @@ public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
URLEncoder.encode(UserGroupInformation.getCurrentUser().getShortUserName(), "UTF8");

HttpURLConnection httpURLConnection =
(HttpURLConnection) (new URL(url + tokenString)).openConnection();
(HttpURLConnection) URI.create(url + tokenString).toURL().openConnection();

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we use new URI instead of URI.create()? based on this there are compile time enforcements ? WDYT?
Image

@abstractdog abstractdog Jun 12, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

personally, I prefer unchecked exceptions; code is always full of try-catch boilerplates, 99% of them are re-throwing the original error wrapped to RuntimeException, because these errors are considered fatal, and the developer has no idea how to handle them (if it needs to be handled at all), so from this point of view, URI.create looks easier and more convenient to me, as it already takes care of the same

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok.

this.connectionConf.configure(httpURLConnection);

return httpURLConnection;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.nio.ByteBuffer;
import java.util.ArrayList;
Expand Down Expand Up @@ -141,7 +142,7 @@ private static List<URL> getDirAndFileURL() throws MalformedURLException {
File lastFile = null;
// Add one file and one directory.
for (String path : classpaths) {
URL url = new URL("file://" + path);
URL url = URI.create("file://" + path).toURL();
File file = FileUtils.toFile(url);
if (lastFile == null) {
lastFile = file;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import static org.junit.Assert.assertTrue;

import java.io.IOException;
import java.net.URI;
import java.net.URL;
import java.util.Collections;

Expand Down Expand Up @@ -77,7 +78,7 @@ public void testAddResourceToClasspath() throws IOException, TezException {
assertTrue(localFs.createNewFile(p));
String urlForm = p.toUri().toURL().toString();
urlForm = urlForm.substring(0, urlForm.lastIndexOf('/') + 1);
URL url = new URL(urlForm);
URL url = URI.create(urlForm).toURL();

ReflectionUtils.addResourcesToSystemClassLoader(Collections.singletonList(url));

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import static org.mockito.Mockito.mock;

import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
Expand Down Expand Up @@ -55,7 +55,7 @@ public void testPseudoAuthenticatorConnectionUrlShouldHaveUserName() throws Exce
.PseudoAuthenticatedURLConnectionFactory(connConf);
String inputUrl = "http://host:8080/path";
String expectedUrl = inputUrl + "?user.name=" + UserGroupInformation.getCurrentUser().getShortUserName();
HttpURLConnection httpURLConnection = connectionFactory.getHttpURLConnection(new URL(inputUrl));
HttpURLConnection httpURLConnection = connectionFactory.getHttpURLConnection(URI.create(inputUrl).toURL());
Assert.assertEquals(expectedUrl, httpURLConnection.getURL().toString());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import java.io.IOException;
import java.io.PrintWriter;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;
Expand Down Expand Up @@ -142,9 +143,9 @@ public void setCorsHeaders() {
String origin = request().getHeader(ORIGIN);
if(origin == null) {
try {
URL url = new URL(historyUrlBase);
URL url = URI.create(historyUrlBase).toURL();
origin = url.getProtocol() + "://" + url.getAuthority();
} catch (MalformedURLException e) {
} catch (IllegalArgumentException | MalformedURLException e) {
LOG.debug("Invalid url set for tez history url base: {}", historyUrlBase, e);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import java.io.IOException;
import java.io.RandomAccessFile;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URL;
import java.nio.channels.ClosedChannelException;
import java.util.ArrayList;
Expand Down Expand Up @@ -427,7 +428,7 @@ public void channelRead(ChannelHandlerContext ctx, Object message)
HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
try {
verifyRequest(jobId, ctx, request, response,
new URL("http", "", this.port, reqUri));
URI.create("http://:" + this.port + reqUri).toURL());
} catch (IOException e) {
LOG.warn("Shuffle failure ", e);
sendError(ctx, e.getMessage(), UNAUTHORIZED);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import java.io.IOException;
import java.io.RandomAccessFile;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.channels.ClosedChannelException;
Expand Down Expand Up @@ -1084,7 +1085,7 @@ private void handleRequest(ChannelHandlerContext ctx, HttpRequest request)
HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
try {
verifyRequest(jobId, ctx, request, response,
new URL("http", "", this.port, reqUri));
URI.create("http://:" + this.port + reqUri).toURL());
} catch (IOException e) {
LOG.warn("Shuffle failure ", e);
sendError(ctx, e.getMessage(), UNAUTHORIZED);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import java.net.HttpURLConnection;
import java.net.SocketAddress;
import java.net.SocketException;
import java.net.URI;
import java.net.URL;
import java.nio.ByteBuffer;
import java.util.ArrayList;
Expand Down Expand Up @@ -392,9 +393,9 @@ protected void sendError(ChannelHandlerContext ctx, String message,

// simulate a reducer that closes early by reading a single shuffle header
// then closing the connection
URL url = new URL("http://127.0.0.1:"
URL url = URI.create("http://127.0.0.1:"
+ shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY)
+ "/mapOutput?job=job_12345_1&dag=1&reduce=1&map=attempt_12345_1_m_1_0");
+ "/mapOutput?job=job_12345_1&dag=1&reduce=1&map=attempt_12345_1_m_1_0").toURL();
HttpURLConnection conn = (HttpURLConnection)url.openConnection();
conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME,
ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
Expand Down Expand Up @@ -522,8 +523,8 @@ protected void sendError(ChannelHandlerContext ctx, String message,
+ shuffleHandler.getConfig().get(
ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY);
URL url =
new URL(shuffleBaseURL + "/mapOutput?job=job_12345_1&dag=1&reduce=1&"
+ "map=attempt_12345_1_m_1_0");
URI.create(shuffleBaseURL + "/mapOutput?job=job_12345_1&dag=1&reduce=1&"
+ "map=attempt_12345_1_m_1_0").toURL();
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME,
ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
Expand All @@ -545,8 +546,8 @@ protected void sendError(ChannelHandlerContext ctx, String message,

// For keepAlive via URL
url =
new URL(shuffleBaseURL + "/mapOutput?job=job_12345_1&dag=1&reduce=1&"
+ "map=attempt_12345_1_m_1_0&keepAlive=true");
URI.create(shuffleBaseURL + "/mapOutput?job=job_12345_1&dag=1&reduce=1&"
+ "map=attempt_12345_1_m_1_0&keepAlive=true").toURL();
conn = (HttpURLConnection) url.openConnection();
conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME,
ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
Expand Down Expand Up @@ -591,8 +592,8 @@ public void testSocketKeepAlive() throws Exception {
+ shuffleHandler.getConfig().get(
ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY);
URL url =
new URL(shuffleBaseURL + "/mapOutput?job=job_12345_1&dag=1&reduce=1&"
+ "map=attempt_12345_1_m_1_0");
URI.create(shuffleBaseURL + "/mapOutput?job=job_12345_1&dag=1&reduce=1&"
+ "map=attempt_12345_1_m_1_0").toURL();
conn = (HttpURLConnection) url.openConnection();
conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME,
ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
Expand Down Expand Up @@ -626,9 +627,9 @@ public void testIncompatibleShuffleVersion() throws Exception {

// simulate a reducer that closes early by reading a single shuffle header
// then closing the connection
URL url = new URL("http://127.0.0.1:"
URL url = URI.create("http://127.0.0.1:"
+ shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY)
+ "/mapOutput?job=job_12345_1&&dag=1reduce=1&map=attempt_12345_1_m_1_0");
+ "/mapOutput?job=job_12345_1&&dag=1reduce=1&map=attempt_12345_1_m_1_0").toURL();
for (int i = 0; i < failureNum; ++i) {
HttpURLConnection conn = (HttpURLConnection)url.openConnection();
conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME,
Expand Down Expand Up @@ -713,7 +714,7 @@ protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx,
+ shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY)
+ "/mapOutput?job=job_12345_1&dag=1&reduce=1&map=attempt_12345_1_m_"
+ i + "_0";
URL url = new URL(URLstring);
URL url = URI.create(URLstring).toURL();
conns[i] = (HttpURLConnection)url.openConnection();
conns[i].setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME,
ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
Expand Down Expand Up @@ -788,12 +789,12 @@ public void testRangedFetch() throws IOException {
appId, ByteBuffer.wrap(outputBuffer.getData(), 0,
outputBuffer.getLength())));
URL url =
new URL(
URI.create(
"http://127.0.0.1:"
+ shuffleHandler.getConfig().get(
ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY)
+ "/mapOutput?job=job_12345_0001&dag=1&reduce=" + reducerIdStart + "-" + reducerIdEnd
+ "&map=attempt_12345_1_m_1_0");
+ "&map=attempt_12345_1_m_1_0").toURL();
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME,
ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
Expand Down Expand Up @@ -886,9 +887,9 @@ private void runMultiAttemptMultiRangeShuffleTest(int attemptRange, int reduceRa
jt.write(outputBuffer);
shuffleHandler.initializeApplication(new ApplicationInitializationContext(user, appId,
ByteBuffer.wrap(outputBuffer.getData(), 0, outputBuffer.getLength())));
URL url = new URL("http://127.0.0.1:" + shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY)
URL url = URI.create("http://127.0.0.1:" + shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY)
+ "/mapOutput?job=job_12345_0001&dag=1&reduce=" + reducerIdStart + "-" + reducerIdEnd + "&map="
+ String.join(",", attemptIds));
+ String.join(",", attemptIds)).toURL();
LOG.info("Calling shuffle URL: {}", url);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME, ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
Expand Down Expand Up @@ -966,12 +967,12 @@ public void testMapFileAccess() throws IOException {
appId, ByteBuffer.wrap(outputBuffer.getData(), 0,
outputBuffer.getLength())));
URL url =
new URL(
URI.create(
"http://127.0.0.1:"
+ shuffleHandler.getConfig().get(
ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY)
+ "/mapOutput?job=job_12345_0001&dag=1&reduce=" + reducerId
+ "&map=attempt_12345_1_m_1_0");
+ "&map=attempt_12345_1_m_1_0").toURL();
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME,
ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
Expand Down Expand Up @@ -1206,10 +1207,10 @@ public void testRecoveryFromOtherVersions() throws IOException {

private static int getShuffleResponseCode(ShuffleHandler shuffle,
Token<JobTokenIdentifier> jt) throws IOException {
URL url = new URL("http://127.0.0.1:"
URL url = URI.create("http://127.0.0.1:"
+ shuffle.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY)
+ "/mapOutput?job=job_12345_0001&dag=1&reduce=0" +
"&map=attempt_12345_1_m_1_0");
"&map=attempt_12345_1_m_1_0").toURL();
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
String encHash = SecureShuffleUtils.hashFromString(
SecureShuffleUtils.buildMsgFrom(url),
Expand Down Expand Up @@ -1303,12 +1304,12 @@ public AuxiliaryLocalPathHandler getAuxiliaryLocalPathHandler() {
appId, ByteBuffer.wrap(outputBuffer.getData(), 0,
outputBuffer.getLength())));
URL url =
new URL(
URI.create(
"http://127.0.0.1:"
+ shuffleHandler.getConfig().get(
ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY)
+ "/mapOutput?job=job_12345_0001&dag=1&reduce=" + reducerId
+ "&map=attempt_12345_1_m_1_0");
+ "&map=attempt_12345_1_m_1_0").toURL();
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME,
ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
Expand Down Expand Up @@ -1380,11 +1381,11 @@ public AuxiliaryLocalPathHandler getAuxiliaryLocalPathHandler() {
appId, ByteBuffer.wrap(outputBuffer.getData(), 0,
outputBuffer.getLength())));
URL url =
new URL(
URI.create(
"http://127.0.0.1:"
+ shuffleHandler.getConfig().get(
ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY)
+ "/mapOutput?dagAction=delete&job=job_12345_0001&dag=1");
+ "/mapOutput?dagAction=delete&job=job_12345_0001&dag=1").toURL();
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME,
ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
Expand Down Expand Up @@ -1472,11 +1473,11 @@ public AuxiliaryLocalPathHandler getAuxiliaryLocalPathHandler() {
appId, ByteBuffer.wrap(outputBuffer.getData(), 0,
outputBuffer.getLength())));
URL url =
new URL(
URI.create(
"http://127.0.0.1:"
+ shuffleHandler.getConfig().get(
ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY)
+ "/mapOutput?vertexAction=delete&job=job_12345_0001&dag=1&vertex=00");
+ "/mapOutput?vertexAction=delete&job=job_12345_0001&dag=1&vertex=00").toURL();
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME,
ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
Expand Down Expand Up @@ -1559,11 +1560,11 @@ public AuxiliaryLocalPathHandler getAuxiliaryLocalPathHandler() {
appId, ByteBuffer.wrap(outputBuffer.getData(), 0,
outputBuffer.getLength())));
URL url =
new URL(
URI.create(
"http://127.0.0.1:"
+ shuffleHandler.getConfig().get(
ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY)
+ "/mapOutput?taskAttemptAction=delete&job=job_12345_0001&dag=1&map=" + appAttemptId);
+ "/mapOutput?taskAttemptAction=delete&job=job_12345_0001&dag=1&map=" + appAttemptId).toURL();
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME,
ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
Expand Down Expand Up @@ -1643,8 +1644,8 @@ public void testShuffleHandlerSendsDiskError() throws Exception {

String shuffleBaseURL = "http://127.0.0.1:"
+ shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY);
URL url = new URL(
shuffleBaseURL + "/mapOutput?job=job_12345_1&dag=1&reduce=1&map=attempt_12345_1_m_1_0");
URL url = URI.create(
shuffleBaseURL + "/mapOutput?job=job_12345_1&dag=1&reduce=1&map=attempt_12345_1_m_1_0").toURL();
shuffleHandler.secretManager.addTokenForJob("job_12345_1",
new Token<>("id".getBytes(), shuffleHandler.getSecret().getBytes(), null, null));

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -345,7 +345,7 @@ static class PseudoAuthenticatedURLConnectionFactory implements HttpURLConnectio
public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
String tokenString = (url.getQuery() == null ? "?" : "&") + "user.name=" +
URLEncoder.encode(UserGroupInformation.getCurrentUser().getShortUserName(), "UTF8");
return (HttpURLConnection) (new URL(url.toString() + tokenString)).openConnection();
return (HttpURLConnection) URI.create(url.toString() + tokenString).toURL().openConnection();
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.nio.ByteBuffer;

Expand Down Expand Up @@ -168,7 +169,7 @@ public static URL constructBaseURIForShuffleHandlerDagComplete(
sb.append(appId.replace("application", "job"));
sb.append("&dag=");
sb.append(dagIdentifier);
return new URL(sb.toString());
return URI.create(sb.toString()).toURL();
}

public static URL constructBaseURIForShuffleHandlerVertexComplete(
Expand All @@ -187,7 +188,7 @@ public static URL constructBaseURIForShuffleHandlerVertexComplete(
sb.append(dagIdentifier);
sb.append("&vertex=");
sb.append(vertexIdentifier);
return new URL(sb.toString());
return URI.create(sb.toString()).toURL();
}

public static URL constructBaseURIForShuffleHandlerTaskAttemptFailed(
Expand All @@ -206,7 +207,7 @@ public static URL constructBaseURIForShuffleHandlerTaskAttemptFailed(
sb.append(dagIdentifier);
sb.append("&map=");
sb.append(taskAttemptIdentifier);
return new URL(sb.toString());
return URI.create(sb.toString()).toURL();
}

public static HttpConnectionParams getHttpConnectionParams(Configuration conf) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import java.io.InputStream;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.nio.ByteBuffer;
import java.text.DecimalFormat;
Expand Down Expand Up @@ -245,7 +246,7 @@ public static URL constructInputURL(String baseURI,
if (keepAlive) {
url.append("&keepAlive=true");
}
return new URL(url.toString());
return URI.create(url.toString()).toURL();
}

public static BaseHttpConnection getHttpConnection(boolean asyncHttp, URL url,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@

import java.io.IOException;
import java.net.ConnectException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.channels.ClosedByInterruptException;
Expand Down Expand Up @@ -76,7 +77,7 @@ public Thread newThread(Runnable r) {
return t;
}
});
url = new URL(NOT_HOSTED_URL);
url = URI.create(NOT_HOSTED_URL).toURL();
tokenSecretManager = mock(JobTokenSecretManager.class);
when(tokenSecretManager.computeHash(any())).thenReturn("1234".getBytes());
}
Expand Down
Loading