Solr4.8.0源码分析(20)之SolrCloud的Recovery策略(一)

 1 private void doFinish() {
 2     // TODO: if not a forward and replication req is not specified, we could
 3     // send in a background thread
 4 
 5     cmdDistrib.finish();
 6     List<Error> errors = cmdDistrib.getErrors();
 7     // TODO - we may need to tell about more than one error...
 8     
 9     // if its a forward, any fail is a problem - 
10     // otherwise we assume things are fine if we got it locally
11     // until we start allowing min replication param
12     if (errors.size() > 0) {
13       // if one node is a RetryNode, this was a forward request
14       if (errors.get(0).req.node instanceof RetryNode) {
15         rsp.setException(errors.get(0).e);
16       } else {
17         if (log.isWarnEnabled()) {
18           for (Error error : errors) {
19             log.warn("Error sending update", error.e);
20           }
21         }
22       }
23       // else
24       // for now we don‘t error - we assume if it was added locally, we
25       // succeeded 
26     }
27    
28     
29     // if it is not a forward request, for each fail, try to tell them to
30     // recover - the doc was already added locally, so it should have been
31     // legit
32 
33     for (final SolrCmdDistributor.Error error : errors) {
34       if (error.req.node instanceof RetryNode) {
35         // we don‘t try to force a leader to recover
36         // when we cannot forward to it
37         continue;
38       }
39       // TODO: we should force their state to recovering ??
40       // TODO: do retries??
41       // TODO: what if its is already recovering? Right now recoveries queue up -
42       // should they?
43       final String recoveryUrl = error.req.node.getBaseUrl();
44       
45       Thread thread = new Thread() {
46         {
47           setDaemon(true);
48         }
49         @Override
50         public void run() {
51           log.info("try and ask " + recoveryUrl + " to recover");
52           HttpSolrServer server = new HttpSolrServer(recoveryUrl);
53           try {
54             server.setSoTimeout(60000);
55             server.setConnectionTimeout(15000);
56             
57             RequestRecovery recoverRequestCmd = new RequestRecovery();
58             recoverRequestCmd.setAction(CoreAdminAction.REQUESTRECOVERY);
59             recoverRequestCmd.setCoreName(error.req.node.getCoreName());
60             try {
61               server.request(recoverRequestCmd);
62             } catch (Throwable t) {
63               SolrException.log(log, recoveryUrl
64                   + ": Could not tell a replica to recover", t);
65             }
66           } finally {
67             server.shutdown();
68           }
69         }
70       };
71       ExecutorService executor = req.getCore().getCoreDescriptor().getCoreContainer().getUpdateShardHandler().getUpdateExecutor();
72       executor.execute(thread);
73       
74     }
75   } 

1       // if true, we are recovering after startup and shouldn‘t have (or be receiving) additional updates (except for local tlog recovery)
2       boolean recoveringAfterStartup = recoveryStrat == null;
3 
4       recoveryStrat = new RecoveryStrategy(cc, cd, this);
5       recoveryStrat.setRecoveringAfterStartup(recoveringAfterStartup);
6       recoveryStrat.start();
7       recoveryRunning = true;

1         zkController.publish(core.getCoreDescriptor(), ZkStateReader.RECOVERING);

 1     if (recoveringAfterStartup) {
 2       // if we‘re recovering after startup (i.e. we have been down), then we need to know what the last versions were
 3       // when we went down.  We may have received updates since then.
 4       recentVersions = startingVersions;
 5       try {
 6         if ((ulog.getStartingOperation() & UpdateLog.FLAG_GAP) != 0) {
 7           // last operation at the time of startup had the GAP flag set...
 8           // this means we were previously doing a full index replication
 9           // that probably didn‘t complete and buffering updates in the
10           // meantime.
11           log.info("Looks like a previous replication recovery did not complete - skipping peer sync. core="
12               + coreName);
13           firstTime = false; // skip peersync
14         }
15       } catch (Exception e) {
16         SolrException.log(log, "Error trying to get ulog starting operation. core="
17             + coreName, e);
18         firstTime = false; // skip peersync
19       }
20     }