-
Notifications
You must be signed in to change notification settings - Fork 852
Tidy up java code in o.a.s.filestore package #4657
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
52a8e0c
0b38d47
442df01
4c02075
c1aac21
5440c0b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -39,7 +39,6 @@ | |
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Objects; | ||
| import java.util.concurrent.ConcurrentHashMap; | ||
| import java.util.function.Consumer; | ||
| import java.util.function.Predicate; | ||
| import java.util.stream.Stream; | ||
|
|
@@ -72,7 +71,6 @@ public class DistribFileStore implements FileStore { | |
|
|
||
| private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); | ||
| private final CoreContainer coreContainer; | ||
| private Map<String, FileInfo> tmpFiles = new ConcurrentHashMap<>(); | ||
|
|
||
| private final Path solrHome; | ||
|
|
||
|
|
@@ -113,13 +111,6 @@ class FileInfo { | |
| this.path = path; | ||
| } | ||
|
|
||
| ByteBuffer getFileData(boolean validate) throws IOException { | ||
| if (fileData == null) { | ||
| fileData = ByteBuffer.wrap(Files.readAllBytes(getRealPath(path))); | ||
| } | ||
| return fileData; | ||
| } | ||
|
|
||
| public String getMetaPath() { | ||
| if (metaPath == null) { | ||
| metaPath = _getMetapath(path); | ||
|
|
@@ -185,7 +176,6 @@ private boolean fetchFileFromNodeAndPersist(String fromNode) { | |
| ByteBuffer metadata; | ||
| Map<?, ?> m; | ||
|
|
||
| InputStream is = null; | ||
| var solrClient = coreContainer.getDefaultHttpSolrClient(); | ||
|
|
||
| try { | ||
|
|
@@ -199,11 +189,9 @@ private boolean fetchFileFromNodeAndPersist(String fromNode) { | |
| } | ||
| } catch (Exception e) { | ||
| throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error fetching metadata", e); | ||
| } finally { | ||
| org.apache.solr.common.util.IOUtils.closeQuietly(is); | ||
| } | ||
|
|
||
| ByteBuffer filedata = null; | ||
| ByteBuffer filedata; | ||
| try { | ||
| final var fileRequest = new FileStoreApi.GetFile(path); | ||
| final var fileResponse = fileRequest.processWithBaseUrl(solrClient, baseUrl, null); | ||
|
|
@@ -227,8 +215,6 @@ private boolean fetchFileFromNodeAndPersist(String fromNode) { | |
| return true; | ||
| } catch (IOException ioe) { | ||
| throw new SolrException(SERVER_ERROR, "Error persisting file", ioe); | ||
| } finally { | ||
| org.apache.solr.common.util.IOUtils.closeQuietly(is); | ||
| } | ||
| } | ||
|
|
||
|
|
@@ -351,7 +337,6 @@ public void put(FileEntry entry) throws IOException { | |
|
|
||
| private void distribute(FileInfo info) { | ||
| try { | ||
| String dirName = info.path.substring(0, info.path.lastIndexOf('/')); | ||
|
|
||
| coreContainer | ||
| .getZkController() | ||
|
|
@@ -367,65 +352,49 @@ private void distribute(FileInfo info) { | |
| } catch (Exception e) { | ||
| throw new SolrException(SERVER_ERROR, "Unable to create an entry in ZK", e); | ||
| } | ||
| tmpFiles.put(info.path, info); | ||
|
|
||
| List<String> nodes = FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer); | ||
| int i = 0; | ||
| int FETCHFROM_SRC = 50; | ||
| String myNodeName = coreContainer.getZkController().getNodeName(); | ||
| String getFrom = ""; | ||
| try { | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @psalagnac can you eyeball one more cahnge? So I looked at what copilot said, and then I eneded up removing the entire outer because at this point, there isn't anything left. Does this all make sense? I'm always nervous around threds ;-)
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This looks good to me. |
||
| for (String node : nodes) { | ||
| String baseUrl = | ||
| coreContainer.getZkController().getZkStateReader().getBaseUrlV2ForNodeName(node); | ||
|
|
||
| String nodeToFetchFrom; | ||
| if (i < FETCHFROM_SRC) { | ||
| // this is to protect very large clusters from overwhelming a single node | ||
| // the first FETCHFROM_SRC nodes will be asked to fetch from this node. | ||
| // it's there in the memory now. So , it must be served fast | ||
| nodeToFetchFrom = myNodeName; | ||
| } else { | ||
| if (i == FETCHFROM_SRC) { | ||
| // This is just an optimization | ||
| // at this point a bunch of nodes are already downloading from me | ||
| // I'll wait for them to finish before asking other nodes to download from each other | ||
| try { | ||
| Thread.sleep(2 * 1000); | ||
| } catch (Exception e) { | ||
| } | ||
| for (String node : nodes) { | ||
| String baseUrl = | ||
| coreContainer.getZkController().getZkStateReader().getBaseUrlV2ForNodeName(node); | ||
|
|
||
| String nodeToFetchFrom; | ||
| if (i < FETCHFROM_SRC) { | ||
| // this is to protect very large clusters from overwhelming a single node | ||
| // the first FETCHFROM_SRC nodes will be asked to fetch from this node. | ||
| // it's there in the memory now. So, it must be served fast | ||
| nodeToFetchFrom = myNodeName; | ||
| } else { | ||
| if (i == FETCHFROM_SRC) { | ||
| // This is just an optimization | ||
| // at this point a bunch of nodes are already downloading from me. | ||
| // I'll wait for them to finish before asking other nodes to download from each other | ||
| try { | ||
| Thread.sleep(2 * 1000); | ||
| } catch (InterruptedException e) { | ||
| Thread.currentThread().interrupt(); | ||
| } | ||
| // trying to avoid the thundering herd problem when there are a very large number of | ||
| // nodes others should try to fetch it from any node where it is available. By now, | ||
| // almost FETCHFROM_SRC other nodes may have it | ||
| nodeToFetchFrom = "*"; | ||
| } | ||
| try { | ||
| final var pullFileRequest = new FileStoreApi.FetchFile(info.path); | ||
| pullFileRequest.setGetFrom(nodeToFetchFrom); | ||
| final var client = coreContainer.getDefaultHttpSolrClient(); | ||
| // fire and forget | ||
| pullFileRequest.processWithBaseUrl(client, baseUrl, null); | ||
| } catch (Exception e) { | ||
| log.info("Node: {} failed to respond for file fetch notification", node, e); | ||
| // ignore the exception | ||
| // some nodes may be down or not responding | ||
| } | ||
| i++; | ||
| // trying to avoid the thundering herd problem when there are a very large number of | ||
| // nodes others should try to fetch it from any node where it is available. By now, | ||
| // almost FETCHFROM_SRC other nodes may have it | ||
| nodeToFetchFrom = "*"; | ||
| } | ||
| } finally { | ||
| coreContainer | ||
| .getUpdateShardHandler() | ||
| .getUpdateExecutor() | ||
| .submit( | ||
| () -> { | ||
| try { | ||
| Thread.sleep(10 * 1000); | ||
| } finally { | ||
| tmpFiles.remove(info.path); | ||
| } | ||
| return null; | ||
| }); | ||
| try { | ||
| final var pullFileRequest = new FileStoreApi.FetchFile(info.path); | ||
| pullFileRequest.setGetFrom(nodeToFetchFrom); | ||
| final var client = coreContainer.getDefaultHttpSolrClient(); | ||
| // fire and forget | ||
| pullFileRequest.processWithBaseUrl(client, baseUrl, null); | ||
| } catch (Exception e) { | ||
| log.info("Node: {} failed to respond for file fetch notification", node, e); | ||
| // ignore the exception | ||
| // some nodes may be down or not responding | ||
| } | ||
| i++; | ||
| } | ||
| } | ||
|
|
||
|
|
@@ -487,7 +456,6 @@ public void syncToAllNodes(String path) throws IOException { | |
| if (!fi.exists(true, false)) { | ||
| throw new SolrException(BAD_REQUEST, "No such file : " + path); | ||
| } | ||
| fi.getFileData(true); | ||
| distribute(fi); | ||
| } | ||
|
|
||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
That's correct. The two buffers are now unused and can be removed.