Revision 46761
Added by Alessia Bardi over 7 years ago
| modules/dnet-data-services/branches/saxonHE/src/main/java/eu/dnetlib/data/collector/plugins/sftp/SftpIterator.java | ||
|---|---|---|
| 23 | 23 |
|
| 24 | 24 |
private static final Log log = LogFactory.getLog(SftpIterator.class); |
| 25 | 25 |
|
| 26 |
private static final int MAX_RETRIES = 5;
|
|
| 26 |
private static final int MAX_RETRIES = 2;
|
|
| 27 | 27 |
private static final int DEFAULT_TIMEOUT = 30000; |
| 28 | 28 |
private static final long BACKOFF_MILLIS = 10000; |
| 29 | 29 |
|
| ... | ... | |
| 150 | 150 |
} |
| 151 | 151 |
|
| 152 | 152 |
private void initializeQueue() {
|
| 153 |
queue = new LinkedList<String>();
|
|
| 153 |
queue = new LinkedList<>(); |
|
| 154 | 154 |
log.info(String.format("SFTP collector plugin collecting from %s with recursion = %s, incremental = %s with fromDate=%s", remoteSftpBasePath,
|
| 155 | 155 |
isRecursive, |
| 156 | 156 |
incremental, fromDate)); |
| ... | ... | |
| 191 | 191 |
Instant instant = Instant.ofEpochMilli(mTime * 1000L); |
| 192 | 192 |
LocalDateTime dt = LocalDateTime.ofInstant(instant, ZoneId.systemDefault()); |
| 193 | 193 |
if (dt.isAfter(fromDate)) {
|
| 194 |
queue.add(currentDir+"/"+currentFileName);
|
|
| 194 |
queue.add(dirToList+"/"+currentFileName);
|
|
| 195 | 195 |
log.debug(dirToList+"/"+currentFileName + " has changed and must be re-collected"); |
| 196 | 196 |
} else {
|
| 197 | 197 |
if (log.isDebugEnabled()) {
|
| ... | ... | |
| 227 | 227 |
public String next() {
|
| 228 | 228 |
String nextRemotePath = queue.remove(); |
| 229 | 229 |
int nRepeat = 0; |
| 230 |
String fullPathFile = nextRemotePath;
|
|
| 230 |
String fullPathFile = "";
|
|
| 231 | 231 |
while (nRepeat < MAX_RETRIES) {
|
| 232 | 232 |
try {
|
| 233 | 233 |
OutputStream baos = new ByteArrayOutputStream(); |
| 234 | 234 |
sftpChannel.get(nextRemotePath, baos); |
| 235 |
fullPathFile = sftpChannel.pwd() + "/" + nextRemotePath; |
|
| 236 | 235 |
if (log.isDebugEnabled()) {
|
| 236 |
fullPathFile = sftpChannel.pwd() + "/" + nextRemotePath; |
|
| 237 | 237 |
log.debug(String.format("Collected file from SFTP: %s%s", sftpServerAddress, fullPathFile));
|
| 238 | 238 |
} |
| 239 | 239 |
return baos.toString(); |
Also available in: Unified diff
less retries and corrected path in the queue for incremental mode